[llvm-branch-commits] [llvm] AMDGPU/GlobalISel: Fix G_UNMERGE_VALUES lowering for extended LLTs (PR #209201)
Petar Avramovic via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Jul 14 07:12:06 PDT 2026
https://github.com/petar-avramovic updated https://github.com/llvm/llvm-project/pull/209201
>From 83e96b55a598c2088cb79a937412548a7f5b075f Mon Sep 17 00:00:00 2001
From: Petar Avramovic <Petar.Avramovic at amd.com>
Date: Mon, 13 Jul 2026 12:47:17 +0200
Subject: [PATCH] AMDGPU/GlobalISel: Fix G_UNMERGE_VALUES lowering for extended
LLTs
Use integer type for bit twiddling instead of scalar.
---
.../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 4 +-
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 6 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll | 2 +-
.../GlobalISel/artifact-combiner-extract.mir | 12 +-
.../artifact-combiner-unmerge-values.mir | 201 +-
.../GlobalISel/combine-fma-add-ext-mul.ll | 88 +-
.../CodeGen/AMDGPU/GlobalISel/dummy-target.ll | 34 +-
.../CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll | 2111 +++++++++++++----
llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll | 6 +-
.../GlobalISel/fmed3-min-max-const-combine.ll | 6 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll | 330 +--
.../AMDGPU/GlobalISel/insertelement.ll | 23 +-
.../AMDGPU/GlobalISel/legalize-abs.mir | 68 +-
.../AMDGPU/GlobalISel/legalize-add.mir | 110 +-
.../AMDGPU/GlobalISel/legalize-and.mir | 19 +-
.../AMDGPU/GlobalISel/legalize-anyext.mir | 18 +-
.../AMDGPU/GlobalISel/legalize-ashr.mir | 216 +-
.../AMDGPU/GlobalISel/legalize-bitcast.mir | 403 ++--
.../AMDGPU/GlobalISel/legalize-bitreverse.mir | 13 +-
.../AMDGPU/GlobalISel/legalize-bswap.mir | 37 +-
.../AMDGPU/GlobalISel/legalize-ctls.mir | 11 +-
.../GlobalISel/legalize-ctlz-zero-poison.mir | 20 +-
.../AMDGPU/GlobalISel/legalize-ctlz.mir | 15 +-
.../AMDGPU/GlobalISel/legalize-ctpop.mir | 11 +-
.../GlobalISel/legalize-cttz-zero-poison.mir | 13 +-
.../AMDGPU/GlobalISel/legalize-cttz.mir | 9 +-
.../AMDGPU/GlobalISel/legalize-extract.mir | 12 +-
.../AMDGPU/GlobalISel/legalize-fabs.mir | 40 +-
.../AMDGPU/GlobalISel/legalize-fadd.mir | 204 +-
.../GlobalISel/legalize-fcanonicalize.mir | 82 +-
.../AMDGPU/GlobalISel/legalize-fceil.mir | 49 +-
.../AMDGPU/GlobalISel/legalize-fcmp.mir | 54 +-
.../AMDGPU/GlobalISel/legalize-fcos.mir | 100 +-
.../AMDGPU/GlobalISel/legalize-fdiv.mir | 224 +-
.../AMDGPU/GlobalISel/legalize-fexp2.mir | 36 +-
.../AMDGPU/GlobalISel/legalize-ffloor.mir | 100 +-
.../AMDGPU/GlobalISel/legalize-flog.mir | 13 +-
.../AMDGPU/GlobalISel/legalize-flog10.mir | 13 +-
.../AMDGPU/GlobalISel/legalize-fma.mir | 276 +--
.../AMDGPU/GlobalISel/legalize-fmad.s16.mir | 732 +++---
.../AMDGPU/GlobalISel/legalize-fmaximum.mir | 26 +-
.../AMDGPU/GlobalISel/legalize-fmaxnum.mir | 230 +-
.../AMDGPU/GlobalISel/legalize-fminimum.mir | 26 +-
.../AMDGPU/GlobalISel/legalize-fminnum.mir | 230 +-
.../AMDGPU/GlobalISel/legalize-fmul.mir | 204 +-
.../AMDGPU/GlobalISel/legalize-fneg.mir | 40 +-
.../AMDGPU/GlobalISel/legalize-fpext.mir | 44 +-
.../AMDGPU/GlobalISel/legalize-fpow.mir | 78 +-
.../AMDGPU/GlobalISel/legalize-fptosi.mir | 20 +-
.../AMDGPU/GlobalISel/legalize-fptoui.mir | 20 +-
.../AMDGPU/GlobalISel/legalize-fptrunc.mir | 7 +-
.../AMDGPU/GlobalISel/legalize-freeze.mir | 22 +-
.../AMDGPU/GlobalISel/legalize-fshl.mir | 234 +-
.../AMDGPU/GlobalISel/legalize-fshr.mir | 234 +-
.../AMDGPU/GlobalISel/legalize-fsin.mir | 100 +-
.../AMDGPU/GlobalISel/legalize-fsqrt.mir | 100 +-
.../AMDGPU/GlobalISel/legalize-fsub.mir | 208 +-
.../AMDGPU/GlobalISel/legalize-icmp.mir | 48 +-
.../AMDGPU/GlobalISel/legalize-insert.mir | 145 +-
.../GlobalISel/legalize-intrinsic-round.mir | 160 +-
.../GlobalISel/legalize-intrinsic-trunc.mir | 49 +-
...legalize-llvm.amdgcn.image.store.2d.d16.ll | 47 +-
.../AMDGPU/GlobalISel/legalize-load-flat.mir | 68 +-
.../GlobalISel/legalize-load-private.mir | 131 +-
.../AMDGPU/GlobalISel/legalize-lshr.mir | 290 +--
.../AMDGPU/GlobalISel/legalize-mul.mir | 110 +-
.../CodeGen/AMDGPU/GlobalISel/legalize-or.mir | 19 +-
.../AMDGPU/GlobalISel/legalize-phi.mir | 95 +-
.../AMDGPU/GlobalISel/legalize-saddo.mir | 133 +-
.../AMDGPU/GlobalISel/legalize-saddsat.mir | 364 +--
.../AMDGPU/GlobalISel/legalize-sdiv.mir | 46 +-
.../AMDGPU/GlobalISel/legalize-select.mir | 37 +-
.../AMDGPU/GlobalISel/legalize-sext-inreg.mir | 100 +-
.../AMDGPU/GlobalISel/legalize-sext.mir | 24 +-
.../AMDGPU/GlobalISel/legalize-shl.mir | 216 +-
.../GlobalISel/legalize-shuffle-vector.mir | 15 +-
.../legalize-shuffle-vector.s16.mir | 33 +-
.../AMDGPU/GlobalISel/legalize-smax.mir | 112 +-
.../AMDGPU/GlobalISel/legalize-smin.mir | 112 +-
.../AMDGPU/GlobalISel/legalize-smulh.mir | 34 +-
.../AMDGPU/GlobalISel/legalize-smulo.mir | 5 +-
.../AMDGPU/GlobalISel/legalize-srem.mir | 46 +-
.../AMDGPU/GlobalISel/legalize-sshlsat.mir | 350 +--
.../AMDGPU/GlobalISel/legalize-ssubo.mir | 133 +-
.../AMDGPU/GlobalISel/legalize-ssubsat.mir | 366 +--
.../GlobalISel/legalize-store-global.mir | 80 +-
.../AMDGPU/GlobalISel/legalize-sub.mir | 110 +-
.../AMDGPU/GlobalISel/legalize-uaddo.mir | 83 +-
.../AMDGPU/GlobalISel/legalize-uaddsat.mir | 276 +--
.../AMDGPU/GlobalISel/legalize-udiv.mir | 54 +-
.../AMDGPU/GlobalISel/legalize-umax.mir | 112 +-
.../AMDGPU/GlobalISel/legalize-umin.mir | 112 +-
.../AMDGPU/GlobalISel/legalize-umulh.mir | 49 +-
.../AMDGPU/GlobalISel/legalize-umulo.mir | 7 +-
.../GlobalISel/legalize-unmerge-values.mir | 124 +-
.../AMDGPU/GlobalISel/legalize-urem.mir | 54 +-
.../AMDGPU/GlobalISel/legalize-ushlsat.mir | 316 +--
.../AMDGPU/GlobalISel/legalize-usubo.mir | 83 +-
.../AMDGPU/GlobalISel/legalize-usubsat.mir | 252 +-
.../GlobalISel/legalize-vector-args-gfx7.mir | 10 +-
.../legalize-vector-args-gfx8-plus.mir | 180 +-
.../AMDGPU/GlobalISel/legalize-xor.mir | 19 +-
.../AMDGPU/GlobalISel/legalize-zext.mir | 30 +-
.../llvm.amdgcn.image.store.2d.d16.ll | 73 +-
...llvm.amdgcn.raw.buffer.store.format.f16.ll | 332 ++-
....amdgcn.raw.ptr.buffer.store.format.f16.ll | 332 ++-
.../llvm.amdgcn.raw.ptr.tbuffer.store.f16.ll | 93 +-
.../llvm.amdgcn.raw.tbuffer.store.f16.ll | 93 +-
...m.amdgcn.struct.buffer.store.format.f16.ll | 105 +-
...dgcn.struct.ptr.buffer.store.format.f16.ll | 105 +-
.../test/CodeGen/AMDGPU/GlobalISel/saddsat.ll | 177 +-
.../AMDGPU/GlobalISel/select-to-fmin-fmax.ll | 80 +-
.../AMDGPU/GlobalISel/shl-ext-reduce.ll | 94 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll | 4 +-
.../test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll | 177 +-
.../AMDGPU/GlobalISel/strict_fma.f16.ll | 545 ++---
.../test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll | 240 +-
.../test/CodeGen/AMDGPU/GlobalISel/usubsat.ll | 223 +-
...ffer-fat-pointers-contents-legalization.ll | 38 +-
.../build-vector-packed-partial-undef.ll | 58 +-
llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll | 59 +-
llvm/test/CodeGen/AMDGPU/fma.f16.ll | 155 +-
llvm/test/CodeGen/AMDGPU/fmaximum.ll | 363 ++-
llvm/test/CodeGen/AMDGPU/fminimum.ll | 363 ++-
llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll | 880 ++++---
.../AMDGPU/fptrunc.v2f16.no.fast.math.ll | 27 +-
llvm/test/CodeGen/AMDGPU/freeze.ll | 1520 +++++++++---
llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll | 31 +-
llvm/test/CodeGen/AMDGPU/llvm.exp.ll | 243 +-
llvm/test/CodeGen/AMDGPU/llvm.exp10.ll | 296 ++-
llvm/test/CodeGen/AMDGPU/llvm.exp2.ll | 165 +-
.../test/CodeGen/AMDGPU/llvm.fptrunc.round.ll | 17 +-
llvm/test/CodeGen/AMDGPU/llvm.frexp.ll | 153 +-
.../CodeGen/AMDGPU/llvm.is.fpclass.f16.ll | 420 ++--
llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll | 318 ++-
llvm/test/CodeGen/AMDGPU/llvm.log.ll | 595 +++--
llvm/test/CodeGen/AMDGPU/llvm.log10.ll | 595 +++--
llvm/test/CodeGen/AMDGPU/llvm.log2.ll | 266 ++-
llvm/test/CodeGen/AMDGPU/llvm.modf.ll | 145 +-
llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll | 127 +-
llvm/test/CodeGen/AMDGPU/mad-mix.ll | 806 +++++--
llvm/test/CodeGen/AMDGPU/roundeven.ll | 57 +-
llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll | 211 +-
llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll | 83 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll | 81 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll | 8 +-
.../test/CodeGen/AMDGPU/vector-reduce-fadd.ll | 8 +-
.../test/CodeGen/AMDGPU/vector-reduce-fmax.ll | 1024 +++++---
.../test/CodeGen/AMDGPU/vector-reduce-fmin.ll | 1024 +++++---
.../CodeGen/AMDGPU/vector-reduce-fminimum.ll | 23 +-
.../test/CodeGen/AMDGPU/vector-reduce-fmul.ll | 8 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll | 219 +-
.../test/CodeGen/AMDGPU/vector-reduce-smax.ll | 71 +-
.../test/CodeGen/AMDGPU/vector-reduce-smin.ll | 71 +-
.../test/CodeGen/AMDGPU/vector-reduce-umax.ll | 60 +-
.../test/CodeGen/AMDGPU/vector-reduce-umin.ll | 62 +-
156 files changed, 15659 insertions(+), 10447 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 36977f0ddaa84..23cd05fc46f21 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -134,12 +134,12 @@ class LegalizerHelper {
LLVM_ABI LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
LLT MoreTy);
- /// Cast the given value to an LLT::scalar with an equivalent size. Returns
+ /// Cast the given value to an LLT::integer with an equivalent size. Returns
/// the register to use if an instruction was inserted. Returns the original
/// register if no coercion was necessary.
//
// This may also fail and return Register() if there is no legal way to cast.
- LLVM_ABI Register coerceToScalar(Register Val);
+ LLVM_ABI Register coerceToInteger(Register Val);
/// Legalize a single operand \p OpIdx of the machine instruction \p MI as a
/// Use by extending the operand's type to \p WideTy using the specified \p
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 1d5d440677a5e..58711a1049514 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -2139,13 +2139,13 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
}
}
-Register LegalizerHelper::coerceToScalar(Register Val) {
+Register LegalizerHelper::coerceToInteger(Register Val) {
LLT Ty = MRI.getType(Val);
if (Ty.isScalar())
return Val;
const DataLayout &DL = MIRBuilder.getDataLayout();
- LLT NewTy = LLT::scalar(Ty.getSizeInBits());
+ LLT NewTy = LLT::integer(Ty.getSizeInBits());
if (Ty.isPointer()) {
if (DL.isNonIntegralAddressSpace(Ty.getAddressSpace()))
return Register();
@@ -9383,7 +9383,7 @@ LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
if (DstTy.isPointer())
return UnableToLegalize; // TODO
- SrcReg = coerceToScalar(SrcReg);
+ SrcReg = coerceToInteger(SrcReg);
if (!SrcReg)
return UnableToLegalize;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
index ca972118c817f..90c6ef0bdde03 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
@@ -2,7 +2,7 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti < %s | FileCheck -check-prefixes=GCN,GFX6 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
; FIXME: regbankcombiner regression, related to:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-extract.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-extract.mir
index 2bc8d9921b005..39e96c5624850 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-extract.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-extract.mir
@@ -401,8 +401,8 @@ body: |
; CHECK: liveins: $vgpr0, $vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](i32)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(<2 x s16>) = COPY $vgpr1
%2:_(<4 x s16>) = G_CONCAT_VECTORS %0, %1
@@ -421,10 +421,10 @@ body: |
; CHECK: liveins: $vgpr0, $vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(<2 x s16>) = COPY $vgpr1
%2:_(<4 x s16>) = G_CONCAT_VECTORS %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-unmerge-values.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-unmerge-values.mir
index acbbb47db84d6..620f2c1ac4e3d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-unmerge-values.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-unmerge-values.mir
@@ -94,8 +94,9 @@ body: |
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG]](s32), [[SEXT_INREG1]](s32)
@@ -135,11 +136,12 @@ body: |
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C2]]
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
%0:_(<2 x s32>) = COPY $vgpr0_vgpr1
%1:_(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -176,9 +178,10 @@ body: |
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BITCAST1]](s32), [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BITCAST1]](i32), [[LSHR]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
%0:_(<2 x s32>) = COPY $vgpr0_vgpr1
%1:_(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -271,15 +274,15 @@ body: |
; CHECK-LABEL: name: test_unmerge_values_s16_of_concat_vectors_v2s16_v2s16
; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[TRUNC]](s16), implicit [[TRUNC1]](s16), implicit [[TRUNC2]](s16), implicit [[TRUNC3]](s16)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(<2 x s16>) = COPY $vgpr1
@@ -419,25 +422,25 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[COPY]](<2 x s32>)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[COPY1]](<2 x s32>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[TRUNC]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR1]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
- ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR2]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[TRUNC1]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST1]](s32)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR3]](s32)
- ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; CHECK-NEXT: [[TRUNC8:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR4]](s32)
- ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C2]](s32)
- ; CHECK-NEXT: [[TRUNC9:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR5]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[TRUNC]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR1]](i32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C2]](i32)
+ ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR2]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[TRUNC1]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR3]](i32)
+ ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; CHECK-NEXT: [[TRUNC8:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR4]](i32)
+ ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C2]](i32)
+ ; CHECK-NEXT: [[TRUNC9:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR5]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[TRUNC2]](s8), implicit [[TRUNC3]](s8), implicit [[TRUNC4]](s8), implicit [[TRUNC5]](s8), implicit [[TRUNC6]](s8), implicit [[TRUNC7]](s8), implicit [[TRUNC8]](s8), implicit [[TRUNC9]](s8)
%0:_(<2 x s32>) = COPY $vgpr0_vgpr1
%1:_(<2 x s32>) = COPY $vgpr2_vgpr3
@@ -541,16 +544,16 @@ body: |
; CHECK-LABEL: name: test_unmerge_values_s128_of_zext_of_concat_vectors
; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[AND]](s32), [[LSHR]](s32)
- ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[AND1]](s32), [[LSHR1]](s32)
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[AND]](s32), [[LSHR]](i32)
+ ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[AND1]](s32), [[LSHR1]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[MV]](s64), implicit [[MV1]](s64)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(<2 x s16>) = COPY $vgpr1
@@ -589,30 +592,30 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[BITCAST1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[BITCAST1]](i32)
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
; CHECK-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR1]](s32), [[BITCAST3]](s32), [[LSHR2]](s32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR1]](i32), [[BITCAST3]](i32), [[LSHR2]](i32)
; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
; CHECK-NEXT: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
- ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST4]](s32), [[LSHR3]](s32), [[BITCAST5]](s32)
+ ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV8]](<2 x s16>)
+ ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST4]](i32), [[LSHR3]](i32), [[BITCAST5]](i32)
; CHECK-NEXT: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST7]], [[C]](s32)
- ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR4]](s32), [[BITCAST7]](s32), [[LSHR5]](s32)
+ ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[UV10]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[UV11]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR4]](i32), [[BITCAST7]](i32), [[LSHR5]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<3 x s32>), implicit [[BUILD_VECTOR1]](<3 x s32>), implicit [[BUILD_VECTOR2]](<3 x s32>), implicit [[BUILD_VECTOR3]](<3 x s32>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
%1:_(<4 x s16>) = COPY $vgpr2_vgpr3
@@ -1178,15 +1181,15 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST1]](s32)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR1]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[TRUNC]](s8), implicit [[TRUNC1]](s8), implicit [[TRUNC2]](s8), implicit [[TRUNC3]](s8)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
%1:_(<4 x s8>) = G_TRUNC %0
@@ -1335,21 +1338,21 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; CHECK-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[TRUNC]](s16), implicit [[TRUNC1]](s16), implicit [[TRUNC2]](s16), implicit [[TRUNC3]](s16), implicit [[TRUNC4]](s16), implicit [[TRUNC5]](s16)
%0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
%1:_(<3 x s16>), %2:_(<3 x s16>) = G_UNMERGE_VALUES %0
@@ -1370,19 +1373,19 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; CHECK-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR1]](s32), [[BITCAST3]](s32), [[LSHR2]](s32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR1]](i32), [[BITCAST3]](i32), [[LSHR2]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[TRUNC]](s16), implicit [[TRUNC1]](s16), implicit [[TRUNC2]](s16), implicit [[BUILD_VECTOR]](<3 x s32>)
%0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
%1:_(<3 x s16>), %2:_(<3 x s16>) = G_UNMERGE_VALUES %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-mul.ll
index afd76d6d545c0..9d102f8856504 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-mul.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-FAST-DENORM %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-FAST-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-FAST-DENORM %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-FAST-DENORM %s
; fold (fadd fast (fpext (fmul fast x, y)), z) -> (fma (fpext x), (fpext y), z)
; fold (fadd fast x, (fpext (fmul fast y, z))) -> (fma (fpext y), (fpext z), x)
@@ -48,17 +48,27 @@ define amdgpu_vs float @test_f16_f32_add_ext_mul_rhs(half inreg %x, half inreg %
define amdgpu_vs <5 x float> @test_5xf16_5xf32_add_ext_mul(<5 x half> inreg %x, <5 x half> inreg %y, <5 x float> inreg %z) {
; GFX9-FAST-DENORM-LABEL: test_5xf16_5xf32_add_ext_mul:
; GFX9-FAST-DENORM: ; %bb.0: ; %.entry
-; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-FAST-DENORM-NEXT: v_pk_mul_f16 v4, s2, v0
-; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-FAST-DENORM-NEXT: v_pk_mul_f16 v3, s1, v0
; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-FAST-DENORM-NEXT: v_pk_mul_f16 v1, s0, v0
-; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v0, v1, 1.0, s6 op_sel_hi:[1,1,0]
-; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v1, v1, 1.0, s7 op_sel:[1,0,0] op_sel_hi:[1,1,0]
-; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v2, v3, 1.0, s8 op_sel_hi:[1,1,0]
-; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v3, v3, 1.0, s9 op_sel:[1,0,0] op_sel_hi:[1,1,0]
-; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v4, v4, 1.0, s10 op_sel_hi:[1,1,0]
+; GFX9-FAST-DENORM-NEXT: v_pk_mul_f16 v0, s0, v0
+; GFX9-FAST-DENORM-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-FAST-DENORM-NEXT: v_pk_mul_f16 v0, s1, v0
+; GFX9-FAST-DENORM-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v0, s5
+; GFX9-FAST-DENORM-NEXT: v_pk_mul_f16 v0, s2, v0
+; GFX9-FAST-DENORM-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9-FAST-DENORM-NEXT: s_lshr_b32 s3, s0, 16
+; GFX9-FAST-DENORM-NEXT: s_lshr_b32 s4, s1, 16
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v2, s8
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v3, s9
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v4, s10
+; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v0, s0, 1.0, v0 op_sel_hi:[1,1,0]
+; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v1, s3, 1.0, v1 op_sel_hi:[1,1,0]
+; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v2, s1, 1.0, v2 op_sel_hi:[1,1,0]
+; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v3, s4, 1.0, v3 op_sel_hi:[1,1,0]
+; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v4, s2, 1.0, v4 op_sel_hi:[1,1,0]
; GFX9-FAST-DENORM-NEXT: ; return to shader part epilog
;
; GFX10-FAST-DENORM-LABEL: test_5xf16_5xf32_add_ext_mul:
@@ -68,10 +78,14 @@ define amdgpu_vs <5 x float> @test_5xf16_5xf32_add_ext_mul(<5 x half> inreg %x,
; GFX10-FAST-DENORM-NEXT: v_mov_b32_e32 v2, s8
; GFX10-FAST-DENORM-NEXT: v_mov_b32_e32 v3, s9
; GFX10-FAST-DENORM-NEXT: v_mov_b32_e32 v4, s10
+; GFX10-FAST-DENORM-NEXT: s_lshr_b32 s11, s0, 16
+; GFX10-FAST-DENORM-NEXT: s_lshr_b32 s12, s1, 16
+; GFX10-FAST-DENORM-NEXT: s_lshr_b32 s13, s3, 16
+; GFX10-FAST-DENORM-NEXT: s_lshr_b32 s14, s4, 16
; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v0, s0, s3, v0 op_sel_hi:[1,1,0]
-; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v1, s0, s3, v1 op_sel:[1,1,0] op_sel_hi:[1,1,0]
+; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v1, s11, s13, v1 op_sel_hi:[1,1,0]
; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v2, s1, s4, v2 op_sel_hi:[1,1,0]
-; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v3, s1, s4, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0]
+; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v3, s12, s14, v3 op_sel_hi:[1,1,0]
; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v4, s2, s5, v4 op_sel_hi:[1,1,0]
; GFX10-FAST-DENORM-NEXT: ; return to shader part epilog
.entry:
@@ -84,18 +98,30 @@ define amdgpu_vs <5 x float> @test_5xf16_5xf32_add_ext_mul(<5 x half> inreg %x,
define amdgpu_vs <6 x float> @test_6xf16_6xf32_add_ext_mul_rhs(<6 x half> inreg %x, <6 x half> inreg %y, <6 x float> inreg %z) {
; GFX9-FAST-DENORM-LABEL: test_6xf16_6xf32_add_ext_mul_rhs:
; GFX9-FAST-DENORM: ; %bb.0: ; %.entry
-; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-FAST-DENORM-NEXT: v_pk_mul_f16 v5, s2, v0
-; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-FAST-DENORM-NEXT: v_pk_mul_f16 v3, s1, v0
; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-FAST-DENORM-NEXT: v_pk_mul_f16 v1, s0, v0
-; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v0, v1, 1.0, s6 op_sel_hi:[1,1,0]
-; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v1, v1, 1.0, s7 op_sel:[1,0,0] op_sel_hi:[1,1,0]
-; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v2, v3, 1.0, s8 op_sel_hi:[1,1,0]
-; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v3, v3, 1.0, s9 op_sel:[1,0,0] op_sel_hi:[1,1,0]
-; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v4, v5, 1.0, s10 op_sel_hi:[1,1,0]
-; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v5, v5, 1.0, s11 op_sel:[1,0,0] op_sel_hi:[1,1,0]
+; GFX9-FAST-DENORM-NEXT: v_pk_mul_f16 v0, s0, v0
+; GFX9-FAST-DENORM-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-FAST-DENORM-NEXT: v_pk_mul_f16 v0, s1, v0
+; GFX9-FAST-DENORM-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v0, s5
+; GFX9-FAST-DENORM-NEXT: v_pk_mul_f16 v0, s2, v0
+; GFX9-FAST-DENORM-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9-FAST-DENORM-NEXT: s_lshr_b32 s3, s0, 16
+; GFX9-FAST-DENORM-NEXT: s_lshr_b32 s4, s1, 16
+; GFX9-FAST-DENORM-NEXT: s_lshr_b32 s5, s2, 16
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v2, s8
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v3, s9
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v4, s10
+; GFX9-FAST-DENORM-NEXT: v_mov_b32_e32 v5, s11
+; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v0, s0, 1.0, v0 op_sel_hi:[1,1,0]
+; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v1, s3, 1.0, v1 op_sel_hi:[1,1,0]
+; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v2, s1, 1.0, v2 op_sel_hi:[1,1,0]
+; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v3, s4, 1.0, v3 op_sel_hi:[1,1,0]
+; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v4, s2, 1.0, v4 op_sel_hi:[1,1,0]
+; GFX9-FAST-DENORM-NEXT: v_mad_mix_f32 v5, s5, 1.0, v5 op_sel_hi:[1,1,0]
; GFX9-FAST-DENORM-NEXT: ; return to shader part epilog
;
; GFX10-FAST-DENORM-LABEL: test_6xf16_6xf32_add_ext_mul_rhs:
@@ -106,12 +132,18 @@ define amdgpu_vs <6 x float> @test_6xf16_6xf32_add_ext_mul_rhs(<6 x half> inreg
; GFX10-FAST-DENORM-NEXT: v_mov_b32_e32 v3, s9
; GFX10-FAST-DENORM-NEXT: v_mov_b32_e32 v4, s10
; GFX10-FAST-DENORM-NEXT: v_mov_b32_e32 v5, s11
+; GFX10-FAST-DENORM-NEXT: s_lshr_b32 s12, s0, 16
+; GFX10-FAST-DENORM-NEXT: s_lshr_b32 s13, s1, 16
+; GFX10-FAST-DENORM-NEXT: s_lshr_b32 s6, s2, 16
+; GFX10-FAST-DENORM-NEXT: s_lshr_b32 s14, s3, 16
+; GFX10-FAST-DENORM-NEXT: s_lshr_b32 s15, s4, 16
+; GFX10-FAST-DENORM-NEXT: s_lshr_b32 s16, s5, 16
; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v0, s0, s3, v0 op_sel_hi:[1,1,0]
-; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v1, s0, s3, v1 op_sel:[1,1,0] op_sel_hi:[1,1,0]
+; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v1, s12, s14, v1 op_sel_hi:[1,1,0]
; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v2, s1, s4, v2 op_sel_hi:[1,1,0]
-; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v3, s1, s4, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0]
+; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v3, s13, s15, v3 op_sel_hi:[1,1,0]
; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v4, s2, s5, v4 op_sel_hi:[1,1,0]
-; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v5, s2, s5, v5 op_sel:[1,1,0] op_sel_hi:[1,1,0]
+; GFX10-FAST-DENORM-NEXT: v_fma_mix_f32 v5, s6, s16, v5 op_sel_hi:[1,1,0]
; GFX10-FAST-DENORM-NEXT: ; return to shader part epilog
.entry:
%a = fmul fast <6 x half> %x, %y
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/dummy-target.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/dummy-target.ll
index f977f0ce373c4..0d06b6068ebd0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/dummy-target.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/dummy-target.ll
@@ -24,20 +24,21 @@ define <2 x i16> @vop3p_add_v2i16(<2 x i16> %arg0) #0 {
; CHECK-NEXT: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x i16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x i16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[BITCAST1]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x i16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x i16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i16) = G_ADD [[TRUNC]], [[TRUNC2]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(i16) = G_ADD [[TRUNC1]], [[TRUNC3]]
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](i16)
; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](i16)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x i16>)
@@ -66,17 +67,18 @@ define <2 x i16> @halfinsts_add_v2i16(<2 x i16> %arg0) #1 {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[COPY]](i32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x i16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x i16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x i16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x i16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[BITCAST1]], [[BITCAST2]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[LSHR]], [[LSHR1]]
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST3]](<2 x i16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
index 5f683a0c0266a..3fb94c54ad450 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
@@ -6,14 +6,14 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=fiji -denormal-fp-math=ieee < %s | FileCheck -check-prefixes=GFX89,GFX8,GFX8-IEEE %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=fiji -denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX89,GFX8,GFX8-FLUSH %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 -denormal-fp-math=ieee < %s | FileCheck -check-prefixes=GFX89,GFX9,GFX9-IEEE %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 -denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX89,GFX9,GFX9-FLUSH %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 -denormal-fp-math=ieee < %s | FileCheck -check-prefixes=GFX89,GFX9,GFX9-IEEE %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 -denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX89,GFX9,GFX9-FLUSH %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 -denormal-fp-math=ieee < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10,GFX10-IEEE %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 -denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10,GFX10-FLUSH %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 -denormal-fp-math=ieee < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10,GFX10-IEEE %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 -denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10,GFX10-FLUSH %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -denormal-fp-math=ieee < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-IEEE %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FLUSH %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -denormal-fp-math=ieee < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-IEEE %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FLUSH %s
define half @v_fdiv_f16(half %a, half %b) {
; GFX6-IEEE-LABEL: v_fdiv_f16:
@@ -856,123 +856,165 @@ define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX9-IEEE-LABEL: v_fdiv_v2f16:
; GFX9-IEEE: ; %bb.0:
; GFX9-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v6, v1
-; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v5, v4
-; GFX9-IEEE-NEXT: v_rcp_f32_e32 v7, v3
-; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v8, v0
-; GFX9-IEEE-NEXT: v_rcp_f32_e32 v9, v6
-; GFX9-IEEE-NEXT: v_mul_f32_e32 v10, v5, v7
-; GFX9-IEEE-NEXT: v_mad_f32 v11, -v3, v10, v5
-; GFX9-IEEE-NEXT: v_mac_f32_e32 v10, v11, v7
-; GFX9-IEEE-NEXT: v_mad_f32 v3, -v3, v10, v5
-; GFX9-IEEE-NEXT: v_mul_f32_e32 v5, v8, v9
-; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v3, v7
-; GFX9-IEEE-NEXT: v_mad_f32 v7, -v6, v5, v8
-; GFX9-IEEE-NEXT: v_mac_f32_e32 v5, v7, v9
-; GFX9-IEEE-NEXT: v_mad_f32 v6, -v6, v5, v8
-; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v9
-; GFX9-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX9-IEEE-NEXT: v_and_b32_e32 v6, 0xff800000, v6
-; GFX9-IEEE-NEXT: v_add_f32_e32 v3, v3, v10
-; GFX9-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v5, v5
-; GFX9-IEEE-NEXT: v_div_fixup_f16 v2, v3, v2, v4
-; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v5, v1, v0
-; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v8, v6
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v5, v2
+; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v7, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v9, v4, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v10, -v2, v9
+; GFX9-IEEE-NEXT: v_add_f32_e32 v10, v10, v4
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v10, v10, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v9, v10, v9
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v9
+; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v8
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v2, v5
+; GFX9-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v9
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v5, v7, v4
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v9, -v8, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v9, v9, v7
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v9, v9, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v5, v9, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v8, -v8, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v7, v8, v7
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v4, v7, v4
+; GFX9-IEEE-NEXT: v_and_b32_e32 v4, 0xff800000, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v4, v6, v3
+; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX9-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-FLUSH-LABEL: v_fdiv_v2f16:
; GFX9-FLUSH: ; %bb.0:
; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v2
-; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v3
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v4, v4
-; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, v0, v3, neg(0) op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, -v1, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v5, v0 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v6, v8, v4
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v5, v7, v3
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v4, v7, v4
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v4, v4, v6
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, v0, v2, neg(0) op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v5, v0 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v3, v6, v3
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, -v1, v5, v0 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v7, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v8, v2
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v7, v6, v4
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v1, v7, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v4, v5, v4
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v4, v4, v7
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v2, v4, v2, v5
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v1, v0
-; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v3, v5
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_fdiv_v2f16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: v_cvt_f32_f16_e32 v4, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX10-NEXT: v_cvt_f32_f16_e32 v8, v0
-; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX10-NEXT: v_rcp_f32_e32 v6, v4
-; GFX10-NEXT: v_cvt_f32_f16_e32 v9, v7
-; GFX10-NEXT: v_rcp_f32_e32 v5, v3
-; GFX10-NEXT: v_fma_mix_f32 v11, v0, v6, neg(0) op_sel_hi:[1,0,0]
-; GFX10-NEXT: v_fma_mix_f32 v10, v0, v5, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX10-NEXT: v_mad_f32 v13, -v4, v11, v8
-; GFX10-NEXT: v_mad_f32 v12, -v3, v10, v9
-; GFX10-NEXT: v_mac_f32_e32 v11, v13, v6
-; GFX10-NEXT: v_mac_f32_e32 v10, v12, v5
-; GFX10-NEXT: v_mad_f32 v4, -v4, v11, v8
-; GFX10-NEXT: v_mad_f32 v3, -v3, v10, v9
-; GFX10-NEXT: v_mul_f32_e32 v4, v4, v6
-; GFX10-NEXT: v_mul_f32_e32 v3, v3, v5
-; GFX10-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX10-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-NEXT: v_add_f32_e32 v4, v4, v11
-; GFX10-NEXT: v_add_f32_e32 v3, v3, v10
-; GFX10-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX10-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX10-NEXT: v_div_fixup_f16 v0, v4, v1, v0
-; GFX10-NEXT: v_div_fixup_f16 v2, v3, v2, v7
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-IEEE-LABEL: v_fdiv_v2f16:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v4, v4
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, v0, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v6, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, -v1, v5, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v8, -v1, v6, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, v0, 1.0, v7 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v8, v0, 1.0, v8 op_sel:[1,0,0] op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v7, v7, v3
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v8, v8, v4
+; GFX10-IEEE-NEXT: v_add_f32_e32 v5, v7, v5
+; GFX10-IEEE-NEXT: v_add_f32_e32 v6, v8, v6
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, -v1, v5, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v8, -v1, v6, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, v0, 1.0, v7 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v8, v0, 1.0, v8 op_sel:[1,0,0] op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v8, v4
+; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-IEEE-NEXT: v_and_b32_e32 v4, 0xff800000, v4
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX10-IEEE-NEXT: v_add_f32_e32 v4, v4, v6
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v3, v1, v0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v4, v2, v5
+; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-FLUSH-LABEL: v_fdiv_v2f16:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-FLUSH-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v8, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v10, v7
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v6, v4
+; GFX10-FLUSH-NEXT: v_fma_mix_f32 v9, v0, v5, neg(0) op_sel_hi:[1,0,0]
+; GFX10-FLUSH-NEXT: v_fma_mix_f32 v11, v0, v6, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-FLUSH-NEXT: v_mad_f32 v12, -v3, v9, v8
+; GFX10-FLUSH-NEXT: v_mad_f32 v13, -v4, v11, v10
+; GFX10-FLUSH-NEXT: v_mac_f32_e32 v9, v12, v5
+; GFX10-FLUSH-NEXT: v_mac_f32_e32 v11, v13, v6
+; GFX10-FLUSH-NEXT: v_mad_f32 v3, -v3, v9, v8
+; GFX10-FLUSH-NEXT: v_mad_f32 v4, -v4, v11, v10
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v4, v4, v6
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v9
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v4, v4, v11
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v1, v0
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v2, v7
+; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fdiv_v2f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX11-NEXT: v_cvt_f32_f16_e32 v4, v1
-; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX11-NEXT: v_rcp_f32_e32 v4, v4
+; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX11-NEXT: v_cvt_f32_f16_e32 v4, v2
; GFX11-NEXT: v_rcp_f32_e32 v3, v3
+; GFX11-NEXT: v_rcp_f32_e32 v4, v4
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_fma_mix_f32 v6, v0, v4, neg(0) op_sel_hi:[1,0,0]
-; GFX11-NEXT: v_fma_mix_f32 v5, v0, v3, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v0 op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_fma_mix_f32 v7, -v1, v5, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_dual_fmac_f32 v6, v8, v4 :: v_dual_fmac_f32 v5, v7, v3
-; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v0 op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_fma_mix_f32 v7, -v1, v5, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_dual_mul_f32 v4, v8, v4 :: v_dual_mul_f32 v3, v7, v3
+; GFX11-NEXT: v_fma_mix_f32 v5, v0, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v6, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v7, -v1, v5, v0 op_sel_hi:[1,0,1]
+; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX11-NEXT: v_dual_fmac_f32 v5, v7, v3 :: v_dual_fmac_f32 v6, v8, v4
+; GFX11-NEXT: v_fma_mix_f32 v7, -v1, v5, v0 op_sel_hi:[1,0,1]
+; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX11-NEXT: v_dual_mul_f32 v3, v7, v3 :: v_dual_mul_f32 v4, v8, v4
; GFX11-NEXT: v_and_b32_e32 v4, 0xff800000, v4
; GFX11-NEXT: v_dual_add_f32 v4, v4, v6 :: v_dual_and_b32 v3, 0xff800000, v3
; GFX11-NEXT: v_add_f32_e32 v3, v3, v5
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX11-NEXT: v_cvt_f16_f32_e32 v4, v4
; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX11-NEXT: v_div_fixup_f16 v0, v4, v1, v0
-; GFX11-NEXT: v_div_fixup_f16 v2, v3, v2, v5
-; GFX11-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX11-NEXT: v_div_fixup_f16 v0, v3, v1, v0
+; GFX11-NEXT: v_div_fixup_f16 v1, v4, v2, v5
+; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv <2 x half> %a, %b
ret <2 x half> %fdiv
@@ -1011,21 +1053,21 @@ define <2 x half> @v_fdiv_v2f16_afn(<2 x half> %a, <2 x half> %b) {
; GFX9-LABEL: v_fdiv_v2f16_afn:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rcp_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rcp_f16_e32 v1, v1
-; GFX9-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX9-NEXT: v_rcp_f16_e32 v2, v1
+; GFX9-NEXT: v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_e32 v2, v0, v2
+; GFX9-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fdiv_v2f16_afn:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rcp_f16_e32 v1, v1
-; GFX10-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX10-NEXT: v_rcp_f16_e32 v2, v1
+; GFX10-NEXT: v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_e32 v2, v0, v2
+; GFX10-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fdiv_v2f16_afn:
@@ -1037,8 +1079,8 @@ define <2 x half> @v_fdiv_v2f16_afn(<2 x half> %a, <2 x half> %b) {
; GFX11-NEXT: v_rcp_f16_e32 v2, v2
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX11-NEXT: v_mul_f16_e32 v2, v3, v2
-; GFX11-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX11-NEXT: v_mul_f16_e32 v1, v3, v2
+; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv afn <2 x half> %a, %b
ret <2 x half> %fdiv
@@ -1160,123 +1202,165 @@ define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX9-IEEE-LABEL: v_fdiv_v2f16_ulp25:
; GFX9-IEEE: ; %bb.0:
; GFX9-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v6, v1
-; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v5, v4
-; GFX9-IEEE-NEXT: v_rcp_f32_e32 v7, v3
-; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v8, v0
-; GFX9-IEEE-NEXT: v_rcp_f32_e32 v9, v6
-; GFX9-IEEE-NEXT: v_mul_f32_e32 v10, v5, v7
-; GFX9-IEEE-NEXT: v_mad_f32 v11, -v3, v10, v5
-; GFX9-IEEE-NEXT: v_mac_f32_e32 v10, v11, v7
-; GFX9-IEEE-NEXT: v_mad_f32 v3, -v3, v10, v5
-; GFX9-IEEE-NEXT: v_mul_f32_e32 v5, v8, v9
-; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v3, v7
-; GFX9-IEEE-NEXT: v_mad_f32 v7, -v6, v5, v8
-; GFX9-IEEE-NEXT: v_mac_f32_e32 v5, v7, v9
-; GFX9-IEEE-NEXT: v_mad_f32 v6, -v6, v5, v8
-; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v9
-; GFX9-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX9-IEEE-NEXT: v_and_b32_e32 v6, 0xff800000, v6
-; GFX9-IEEE-NEXT: v_add_f32_e32 v3, v3, v10
-; GFX9-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v5, v5
-; GFX9-IEEE-NEXT: v_div_fixup_f16 v2, v3, v2, v4
-; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v5, v1, v0
-; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v8, v6
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v5, v2
+; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v7, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v9, v4, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v10, -v2, v9
+; GFX9-IEEE-NEXT: v_add_f32_e32 v10, v10, v4
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v10, v10, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v9, v10, v9
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v9
+; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v8
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v2, v5
+; GFX9-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v9
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v5, v7, v4
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v9, -v8, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v9, v9, v7
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v9, v9, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v5, v9, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v8, -v8, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v7, v8, v7
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v4, v7, v4
+; GFX9-IEEE-NEXT: v_and_b32_e32 v4, 0xff800000, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v4, v6, v3
+; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX9-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-FLUSH-LABEL: v_fdiv_v2f16_ulp25:
; GFX9-FLUSH: ; %bb.0:
; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v2
-; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v3
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v4, v4
-; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, v0, v3, neg(0) op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, -v1, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v5, v0 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v6, v8, v4
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v5, v7, v3
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v4, v7, v4
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v4, v4, v6
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, v0, v2, neg(0) op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v5, v0 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v3, v6, v3
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, -v1, v5, v0 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v7, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v8, v2
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v7, v6, v4
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v1, v7, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v4, v5, v4
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v4, v4, v7
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v2, v4, v2, v5
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v1, v0
-; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v3, v5
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_fdiv_v2f16_ulp25:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: v_cvt_f32_f16_e32 v4, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX10-NEXT: v_cvt_f32_f16_e32 v8, v0
-; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX10-NEXT: v_rcp_f32_e32 v6, v4
-; GFX10-NEXT: v_cvt_f32_f16_e32 v9, v7
-; GFX10-NEXT: v_rcp_f32_e32 v5, v3
-; GFX10-NEXT: v_fma_mix_f32 v11, v0, v6, neg(0) op_sel_hi:[1,0,0]
-; GFX10-NEXT: v_fma_mix_f32 v10, v0, v5, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX10-NEXT: v_mad_f32 v13, -v4, v11, v8
-; GFX10-NEXT: v_mad_f32 v12, -v3, v10, v9
-; GFX10-NEXT: v_mac_f32_e32 v11, v13, v6
-; GFX10-NEXT: v_mac_f32_e32 v10, v12, v5
-; GFX10-NEXT: v_mad_f32 v4, -v4, v11, v8
-; GFX10-NEXT: v_mad_f32 v3, -v3, v10, v9
-; GFX10-NEXT: v_mul_f32_e32 v4, v4, v6
-; GFX10-NEXT: v_mul_f32_e32 v3, v3, v5
-; GFX10-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX10-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-NEXT: v_add_f32_e32 v4, v4, v11
-; GFX10-NEXT: v_add_f32_e32 v3, v3, v10
-; GFX10-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX10-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX10-NEXT: v_div_fixup_f16 v0, v4, v1, v0
-; GFX10-NEXT: v_div_fixup_f16 v2, v3, v2, v7
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-IEEE-LABEL: v_fdiv_v2f16_ulp25:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v4, v4
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, v0, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v6, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, -v1, v5, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v8, -v1, v6, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, v0, 1.0, v7 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v8, v0, 1.0, v8 op_sel:[1,0,0] op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v7, v7, v3
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v8, v8, v4
+; GFX10-IEEE-NEXT: v_add_f32_e32 v5, v7, v5
+; GFX10-IEEE-NEXT: v_add_f32_e32 v6, v8, v6
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, -v1, v5, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v8, -v1, v6, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, v0, 1.0, v7 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v8, v0, 1.0, v8 op_sel:[1,0,0] op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v8, v4
+; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-IEEE-NEXT: v_and_b32_e32 v4, 0xff800000, v4
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX10-IEEE-NEXT: v_add_f32_e32 v4, v4, v6
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v3, v1, v0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v4, v2, v5
+; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-FLUSH-LABEL: v_fdiv_v2f16_ulp25:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-FLUSH-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v8, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v10, v7
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v6, v4
+; GFX10-FLUSH-NEXT: v_fma_mix_f32 v9, v0, v5, neg(0) op_sel_hi:[1,0,0]
+; GFX10-FLUSH-NEXT: v_fma_mix_f32 v11, v0, v6, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-FLUSH-NEXT: v_mad_f32 v12, -v3, v9, v8
+; GFX10-FLUSH-NEXT: v_mad_f32 v13, -v4, v11, v10
+; GFX10-FLUSH-NEXT: v_mac_f32_e32 v9, v12, v5
+; GFX10-FLUSH-NEXT: v_mac_f32_e32 v11, v13, v6
+; GFX10-FLUSH-NEXT: v_mad_f32 v3, -v3, v9, v8
+; GFX10-FLUSH-NEXT: v_mad_f32 v4, -v4, v11, v10
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v4, v4, v6
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v9
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v4, v4, v11
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v1, v0
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v2, v7
+; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fdiv_v2f16_ulp25:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX11-NEXT: v_cvt_f32_f16_e32 v4, v1
-; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX11-NEXT: v_rcp_f32_e32 v4, v4
+; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX11-NEXT: v_cvt_f32_f16_e32 v4, v2
; GFX11-NEXT: v_rcp_f32_e32 v3, v3
+; GFX11-NEXT: v_rcp_f32_e32 v4, v4
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_fma_mix_f32 v6, v0, v4, neg(0) op_sel_hi:[1,0,0]
-; GFX11-NEXT: v_fma_mix_f32 v5, v0, v3, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v0 op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_fma_mix_f32 v7, -v1, v5, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_dual_fmac_f32 v6, v8, v4 :: v_dual_fmac_f32 v5, v7, v3
-; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v0 op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_fma_mix_f32 v7, -v1, v5, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_dual_mul_f32 v4, v8, v4 :: v_dual_mul_f32 v3, v7, v3
+; GFX11-NEXT: v_fma_mix_f32 v5, v0, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v6, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v7, -v1, v5, v0 op_sel_hi:[1,0,1]
+; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX11-NEXT: v_dual_fmac_f32 v5, v7, v3 :: v_dual_fmac_f32 v6, v8, v4
+; GFX11-NEXT: v_fma_mix_f32 v7, -v1, v5, v0 op_sel_hi:[1,0,1]
+; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX11-NEXT: v_dual_mul_f32 v3, v7, v3 :: v_dual_mul_f32 v4, v8, v4
; GFX11-NEXT: v_and_b32_e32 v4, 0xff800000, v4
; GFX11-NEXT: v_dual_add_f32 v4, v4, v6 :: v_dual_and_b32 v3, 0xff800000, v3
; GFX11-NEXT: v_add_f32_e32 v3, v3, v5
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX11-NEXT: v_cvt_f16_f32_e32 v4, v4
; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX11-NEXT: v_div_fixup_f16 v0, v4, v1, v0
-; GFX11-NEXT: v_div_fixup_f16 v2, v3, v2, v5
-; GFX11-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX11-NEXT: v_div_fixup_f16 v0, v3, v1, v0
+; GFX11-NEXT: v_div_fixup_f16 v1, v4, v2, v5
+; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv <2 x half> %a, %b
ret <2 x half> %fdiv
@@ -1364,29 +1448,151 @@ define <2 x half> @v_rcp_v2f16(<2 x half> %x) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_rcp_v2f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rcp_f16_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-IEEE-LABEL: v_rcp_v2f16:
+; GFX9-IEEE: ; %bb.0:
+; GFX9-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v1
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v6, -v1, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v6, 1.0, v6
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v7, -v3, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v6, v6, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v7, 1.0, v7
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v1, -v1, v6
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v1, 1.0, v1
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v4, v7, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, 1.0, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX9-IEEE-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX9-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX9-IEEE-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
+; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-IEEE-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_rcp_v2f16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rcp_f16_e32 v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-FLUSH-LABEL: v_rcp_v2f16:
+; GFX9-FLUSH: ; %bb.0:
+; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX9-FLUSH-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v0, v1, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v0, v3, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_f32 v5, v5, v1, v1
+; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v0, v6, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v1, v7, v1
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v1, v1, v5
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-IEEE-LABEL: v_rcp_v2f16:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v0, v2, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, -v0, v3, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_add_f32_e32 v4, 1.0, v4
+; GFX10-IEEE-NEXT: v_add_f32_e32 v5, 1.0, v5
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v4, v2
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v5, v5, v3
+; GFX10-IEEE-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX10-IEEE-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v6, -v0, v4, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, -v0, v5, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_add_f32_e32 v6, 1.0, v6
+; GFX10-IEEE-NEXT: v_add_f32_e32 v7, 1.0, v7
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v6, v2
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
+; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-FLUSH-LABEL: v_rcp_v2f16:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, -v2, v4, 1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, -v3, v5, 1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, v6, v4, v4
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, v7, v5, v5
+; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v6, 1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, 1.0
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
+; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_rcp_v2f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-NEXT: v_rcp_f16_e32 v0, v0
-; GFX11-NEXT: v_rcp_f16_e32 v1, v1
+; GFX11-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX11-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX11-NEXT: v_rcp_f32_e32 v2, v2
+; GFX11-NEXT: v_rcp_f32_e32 v3, v3
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_fma_mix_f32 v5, -v0, v2, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v6, -v0, v3, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_f32 v5, v5, v2, v2
+; GFX11-NEXT: v_fma_f32 v6, v6, v3, v3
+; GFX11-NEXT: v_fma_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v4, -v0, v6, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_dual_mul_f32 v2, v7, v2 :: v_dual_mul_f32 v3, v4, v3
+; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX11-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_and_b32 v3, 0xff800000, v3
+; GFX11-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
+; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv <2 x half> <half 1.0, half 1.0>, %x
@@ -1475,29 +1681,151 @@ define <2 x half> @v_neg_rcp_v2f16(<2 x half> %x) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_neg_rcp_v2f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rcp_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rcp_f16_e64 v0, -v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-IEEE-LABEL: v_neg_rcp_v2f16:
+; GFX9-IEEE: ; %bb.0:
+; GFX9-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v1
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v1, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v6, -1.0, v6
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v7, v3, v5
+; GFX9-IEEE-NEXT: v_sub_f32_e32 v6, v6, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v7, -1.0, v7
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v1, -v1, v6
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v1, -1.0, v1
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX9-IEEE-NEXT: v_sub_f32_e32 v4, v7, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, -1.0, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX9-IEEE-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX9-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX9-IEEE-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v1, v0, -1.0
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v3, v2, -1.0
+; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-IEEE-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_neg_rcp_v2f16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rcp_f16_e64 v0, -v0
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-FLUSH-LABEL: v_neg_rcp_v2f16:
+; GFX9-FLUSH: ; %bb.0:
+; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX9-FLUSH-NEXT: v_mov_b32_e32 v4, -1.0
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, v0, v1, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, v0, v3, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_f32 v5, v5, v1, -v1
+; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v3, -v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v0, v6, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v1, v7, v1
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v1, v1, v5
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v0, -1.0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v2, -1.0
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-IEEE-LABEL: v_neg_rcp_v2f16:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, v2, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, v0, v3, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_add_f32_e32 v4, -1.0, v4
+; GFX10-IEEE-NEXT: v_add_f32_e32 v5, -1.0, v5
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v4, v2
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v5, v5, v3
+; GFX10-IEEE-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX10-IEEE-NEXT: v_sub_f32_e32 v5, v5, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v6, -v0, v4, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, -v0, v5, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_add_f32_e32 v6, -1.0, v6
+; GFX10-IEEE-NEXT: v_add_f32_e32 v7, -1.0, v7
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v6, v2
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
+; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-FLUSH-LABEL: v_neg_rcp_v2f16:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, v2, v4, -1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, v3, v5, -1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, v6, v4, -v4
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, v7, v5, -v5
+; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v6, -1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, -1.0
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
+; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_neg_rcp_v2f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-NEXT: v_rcp_f16_e64 v0, -v0
-; GFX11-NEXT: v_rcp_f16_e64 v1, -v1
+; GFX11-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX11-NEXT: v_mov_b32_e32 v4, -1.0
+; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX11-NEXT: v_rcp_f32_e32 v2, v2
+; GFX11-NEXT: v_rcp_f32_e32 v3, v3
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_fma_mix_f32 v5, v0, v2, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v6, v0, v3, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_f32 v5, v5, v2, -v2
+; GFX11-NEXT: v_fma_f32 v6, v6, v3, -v3
+; GFX11-NEXT: v_fma_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v4, -v0, v6, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_dual_mul_f32 v2, v7, v2 :: v_dual_mul_f32 v3, v4, v3
+; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX11-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_and_b32 v3, 0xff800000, v3
+; GFX11-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
+; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv <2 x half> <half -1.0, half -1.0>, %x
@@ -1586,30 +1914,160 @@ define <2 x half> @v_rcp_v2f16_fabs(<2 x half> %x) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_rcp_v2f16_fabs:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rcp_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rcp_f16_e64 v0, |v0|
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-IEEE-LABEL: v_rcp_v2f16_fabs:
+; GFX9-IEEE: ; %bb.0:
+; GFX9-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-IEEE-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v1
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v6, -v1, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v6, 1.0, v6
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v7, -v3, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v6, v6, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v7, 1.0, v7
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v1, -v1, v6
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v1, 1.0, v1
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v4, v7, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, 1.0, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX9-IEEE-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX9-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX9-IEEE-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
+; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-IEEE-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_rcp_v2f16_fabs:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rcp_f16_e64 v0, |v0|
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-FLUSH-LABEL: v_rcp_v2f16_fabs:
+; GFX9-FLUSH: ; %bb.0:
+; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v3
+; GFX9-FLUSH-NEXT: v_mov_b32_e32 v5, 1.0
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v4, v4
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v2, v5 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -|v0|, v4, v5 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v2, v2
+; GFX9-FLUSH-NEXT: v_mad_f32 v7, v7, v4, v4
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, -v1, v6, v5 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v0, -|v0|, v7, v5 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v8, v2
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v0, v0, v7
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v3, 1.0
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-IEEE-LABEL: v_rcp_v2f16_fabs:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-IEEE-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v4, v4
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, -v1, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v6, -|v0|, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_add_f32_e32 v5, 1.0, v5
+; GFX10-IEEE-NEXT: v_add_f32_e32 v6, 1.0, v6
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v5, v5, v3
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX10-IEEE-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX10-IEEE-NEXT: v_add_f32_e32 v6, v6, v4
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, -v1, v5, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v0, -|v0|, v6, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_add_f32_e32 v7, 1.0, v7
+; GFX10-IEEE-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-IEEE-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX10-IEEE-NEXT: v_add_f32_e32 v0, v0, v6
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v0, v2, 1.0
+; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-FLUSH-LABEL: v_rcp_v2f16_fabs:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX10-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, -v2, v4, 1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, -v3, v5, 1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, v6, v4, v4
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, v7, v5, v5
+; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v6, 1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, 1.0
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
+; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_rcp_v2f16_fabs:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-NEXT: v_rcp_f16_e64 v0, |v0|
-; GFX11-NEXT: v_rcp_f16_e64 v1, |v1|
+; GFX11-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX11-NEXT: v_rcp_f32_e32 v4, v4
+; GFX11-NEXT: v_mov_b32_e32 v5, 1.0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX11-NEXT: v_fma_mix_f32 v7, -|v0|, v4, v5 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_f32 v7, v7, v4, v4
+; GFX11-NEXT: v_fma_mix_f32 v0, -|v0|, v7, v5 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX11-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v7
+; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX11-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX11-NEXT: v_rcp_f32_e32 v3, v3
+; GFX11-NEXT: v_div_fixup_f16 v0, v0, v2, 1.0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_fma_mix_f32 v6, -v1, v3, v5 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_f32 v6, v6, v3, v3
+; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v5 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_mul_f32_e32 v3, v8, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX11-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
+; GFX11-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%x.fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %x)
%fdiv = fdiv <2 x half> <half 1.0, half 1.0>, %x.fabs
@@ -1698,30 +2156,160 @@ define <2 x half> @v_neg_rcp_v2f16_fabs(<2 x half> %x) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_neg_rcp_v2f16_fabs:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rcp_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rcp_f16_e64 v0, -|v0|
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-IEEE-LABEL: v_neg_rcp_v2f16_fabs:
+; GFX9-IEEE: ; %bb.0:
+; GFX9-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-IEEE-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v1
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v1, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v6, -1.0, v6
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v7, v3, v5
+; GFX9-IEEE-NEXT: v_sub_f32_e32 v6, v6, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v7, -1.0, v7
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v1, -v1, v6
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v1, -1.0, v1
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX9-IEEE-NEXT: v_sub_f32_e32 v4, v7, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, -1.0, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX9-IEEE-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX9-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX9-IEEE-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v1, v0, -1.0
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v3, v2, -1.0
+; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-IEEE-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_neg_rcp_v2f16_fabs:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rcp_f16_e64 v0, -|v0|
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-FLUSH-LABEL: v_neg_rcp_v2f16_fabs:
+; GFX9-FLUSH: ; %bb.0:
+; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v3
+; GFX9-FLUSH-NEXT: v_mov_b32_e32 v5, -1.0
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v4, v4
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, v1, v2, v5 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, |v0|, v4, v5 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v2, -v2
+; GFX9-FLUSH-NEXT: v_mad_f32 v7, v7, v4, -v4
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, -v1, v6, v5 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v0, -|v0|, v7, v5 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v8, v2
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v0, v0, v7
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v3, -1.0
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-IEEE-LABEL: v_neg_rcp_v2f16_fabs:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-IEEE-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v4, v4
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, v1, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v6, |v0|, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_add_f32_e32 v5, -1.0, v5
+; GFX10-IEEE-NEXT: v_add_f32_e32 v6, -1.0, v6
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v5, v5, v3
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX10-IEEE-NEXT: v_sub_f32_e32 v5, v5, v3
+; GFX10-IEEE-NEXT: v_sub_f32_e32 v6, v6, v4
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, -v1, v5, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v0, -|v0|, v6, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_add_f32_e32 v7, -1.0, v7
+; GFX10-IEEE-NEXT: v_add_f32_e32 v0, -1.0, v0
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-IEEE-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX10-IEEE-NEXT: v_add_f32_e32 v0, v0, v6
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v0, v2, -1.0
+; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-FLUSH-LABEL: v_neg_rcp_v2f16_fabs:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX10-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, v2, v4, -1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, v3, v5, -1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, v6, v4, -v4
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, v7, v5, -v5
+; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v6, -1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, -1.0
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
+; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_neg_rcp_v2f16_fabs:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-NEXT: v_rcp_f16_e64 v0, -|v0|
-; GFX11-NEXT: v_rcp_f16_e64 v1, -|v1|
+; GFX11-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX11-NEXT: v_rcp_f32_e32 v4, v4
+; GFX11-NEXT: v_mov_b32_e32 v5, -1.0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX11-NEXT: v_fma_mix_f32 v7, |v0|, v4, v5 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_f32 v7, v7, v4, -v4
+; GFX11-NEXT: v_fma_mix_f32 v0, -|v0|, v7, v5 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX11-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v7
+; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX11-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX11-NEXT: v_rcp_f32_e32 v3, v3
+; GFX11-NEXT: v_div_fixup_f16 v0, v0, v2, -1.0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_fma_mix_f32 v6, v1, v3, v5 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_f32 v6, v6, v3, -v3
+; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v5 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_mul_f32_e32 v3, v8, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX11-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
+; GFX11-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%x.fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %x)
%fdiv = fdiv <2 x half> <half -1.0, half -1.0>, %x.fabs
@@ -1813,17 +2401,17 @@ define <2 x half> @v_rcp_v2f16_arcp(<2 x half> %x) {
; GFX9-LABEL: v_rcp_v2f16_arcp:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rcp_f16_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-NEXT: v_rcp_f16_e32 v1, v0
+; GFX9-NEXT: v_rcp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_rcp_v2f16_arcp:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rcp_f16_e32 v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-NEXT: v_rcp_f16_e32 v1, v0
+; GFX10-NEXT: v_rcp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_rcp_v2f16_arcp:
@@ -1865,17 +2453,17 @@ define <2 x half> @v_rcp_v2f16_arcp_afn(<2 x half> %x) {
; GFX9-LABEL: v_rcp_v2f16_arcp_afn:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rcp_f16_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-NEXT: v_rcp_f16_e32 v1, v0
+; GFX9-NEXT: v_rcp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_rcp_v2f16_arcp_afn:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rcp_f16_e32 v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-NEXT: v_rcp_f16_e32 v1, v0
+; GFX10-NEXT: v_rcp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_rcp_v2f16_arcp_afn:
@@ -1973,29 +2561,151 @@ define <2 x half> @v_rcp_v2f16_ulp25(<2 x half> %x) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_rcp_v2f16_ulp25:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rcp_f16_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-IEEE-LABEL: v_rcp_v2f16_ulp25:
+; GFX9-IEEE: ; %bb.0:
+; GFX9-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX9-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v1
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v6, -v1, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v6, 1.0, v6
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v7, -v3, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v6, v6, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v7, 1.0, v7
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v1, -v1, v6
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v1, 1.0, v1
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v4, v7, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v4
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, 1.0, v3
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX9-IEEE-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX9-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX9-IEEE-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
+; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-IEEE-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_rcp_v2f16_ulp25:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rcp_f16_e32 v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-FLUSH-LABEL: v_rcp_v2f16_ulp25:
+; GFX9-FLUSH: ; %bb.0:
+; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX9-FLUSH-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v1, v1
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v0, v1, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v0, v3, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_f32 v5, v5, v1, v1
+; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v0, v6, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v1, v7, v1
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v1, v1, v5
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-IEEE-LABEL: v_rcp_v2f16_ulp25:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v0, v2, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, -v0, v3, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_add_f32_e32 v4, 1.0, v4
+; GFX10-IEEE-NEXT: v_add_f32_e32 v5, 1.0, v5
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v4, v2
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v5, v5, v3
+; GFX10-IEEE-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX10-IEEE-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v6, -v0, v4, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, -v0, v5, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_add_f32_e32 v6, 1.0, v6
+; GFX10-IEEE-NEXT: v_add_f32_e32 v7, 1.0, v7
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v6, v2
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
+; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-FLUSH-LABEL: v_rcp_v2f16_ulp25:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, -v2, v4, 1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, -v3, v5, 1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, v6, v4, v4
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, v7, v5, v5
+; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v6, 1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, 1.0
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
+; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_rcp_v2f16_ulp25:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-NEXT: v_rcp_f16_e32 v0, v0
-; GFX11-NEXT: v_rcp_f16_e32 v1, v1
+; GFX11-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX11-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX11-NEXT: v_rcp_f32_e32 v2, v2
+; GFX11-NEXT: v_rcp_f32_e32 v3, v3
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_fma_mix_f32 v5, -v0, v2, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v6, -v0, v3, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_f32 v5, v5, v2, v2
+; GFX11-NEXT: v_fma_f32 v6, v6, v3, v3
+; GFX11-NEXT: v_fma_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v4, -v0, v6, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_dual_mul_f32 v2, v7, v2 :: v_dual_mul_f32 v3, v4, v3
+; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX11-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_and_b32 v3, 0xff800000, v3
+; GFX11-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
+; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv <2 x half> <half 1.0, half 1.0>, %x
@@ -2035,21 +2745,21 @@ define <2 x half> @v_fdiv_v2f16_afn_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX9-LABEL: v_fdiv_v2f16_afn_ulp25:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rcp_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rcp_f16_e32 v1, v1
-; GFX9-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX9-NEXT: v_rcp_f16_e32 v2, v1
+; GFX9-NEXT: v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_e32 v2, v0, v2
+; GFX9-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fdiv_v2f16_afn_ulp25:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rcp_f16_e32 v1, v1
-; GFX10-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX10-NEXT: v_rcp_f16_e32 v2, v1
+; GFX10-NEXT: v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_e32 v2, v0, v2
+; GFX10-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fdiv_v2f16_afn_ulp25:
@@ -2061,8 +2771,8 @@ define <2 x half> @v_fdiv_v2f16_afn_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX11-NEXT: v_rcp_f16_e32 v2, v2
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX11-NEXT: v_mul_f16_e32 v2, v3, v2
-; GFX11-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX11-NEXT: v_mul_f16_e32 v1, v3, v2
+; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv afn <2 x half> %a, %b
ret <2 x half> %fdiv
@@ -2161,21 +2871,21 @@ define <2 x half> @v_fdiv_v2f16_arcp_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX9-LABEL: v_fdiv_v2f16_arcp_ulp25:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rcp_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rcp_f16_e32 v1, v1
-; GFX9-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX9-NEXT: v_rcp_f16_e32 v2, v1
+; GFX9-NEXT: v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_e32 v2, v0, v2
+; GFX9-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fdiv_v2f16_arcp_ulp25:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rcp_f16_e32 v1, v1
-; GFX10-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX10-NEXT: v_rcp_f16_e32 v2, v1
+; GFX10-NEXT: v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_e32 v2, v0, v2
+; GFX10-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fdiv_v2f16_arcp_ulp25:
@@ -2187,8 +2897,8 @@ define <2 x half> @v_fdiv_v2f16_arcp_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX11-NEXT: v_rcp_f16_e32 v2, v2
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX11-NEXT: v_mul_f16_e32 v2, v3, v2
-; GFX11-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX11-NEXT: v_mul_f16_e32 v1, v3, v2
+; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv arcp <2 x half> %a, %b
ret <2 x half> %fdiv
@@ -2227,21 +2937,21 @@ define <2 x half> @v_fdiv_v2f16_arcp_afn_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX9-LABEL: v_fdiv_v2f16_arcp_afn_ulp25:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rcp_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rcp_f16_e32 v1, v1
-; GFX9-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX9-NEXT: v_rcp_f16_e32 v2, v1
+; GFX9-NEXT: v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_mul_f16_e32 v2, v0, v2
+; GFX9-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fdiv_v2f16_arcp_afn_ulp25:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rcp_f16_e32 v1, v1
-; GFX10-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX10-NEXT: v_rcp_f16_e32 v2, v1
+; GFX10-NEXT: v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_mul_f16_e32 v2, v0, v2
+; GFX10-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fdiv_v2f16_arcp_afn_ulp25:
@@ -2253,8 +2963,8 @@ define <2 x half> @v_fdiv_v2f16_arcp_afn_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX11-NEXT: v_rcp_f16_e32 v2, v2
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX11-NEXT: v_mul_f16_e32 v2, v3, v2
-; GFX11-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX11-NEXT: v_mul_f16_e32 v1, v3, v2
+; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv afn arcp <2 x half> %a, %b
ret <2 x half> %fdiv
@@ -2679,129 +3389,185 @@ define amdgpu_ps i32 @s_fdiv_v2f16(i32 inreg %a.arg, i32 inreg %b.arg) {
;
; GFX9-IEEE-LABEL: s_fdiv_v2f16:
; GFX9-IEEE: ; %bb.0:
-; GFX9-IEEE-NEXT: s_lshr_b32 s2, s1, 16
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s1
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s0
+; GFX9-IEEE-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v4, s3
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v2, v0
+; GFX9-IEEE-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-IEEE-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v5, v1, v2
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v6, -v0, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v2
+; GFX9-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v0, -v0, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX9-IEEE-NEXT: v_readfirstlane_b32 s1, v0
; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s2
-; GFX9-IEEE-NEXT: s_lshr_b32 s3, s0, 16
-; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v2, s1
-; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s3
-; GFX9-IEEE-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v4, s0
-; GFX9-IEEE-NEXT: v_rcp_f32_e32 v5, v2
-; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v1, v3
-; GFX9-IEEE-NEXT: v_mad_f32 v7, -v0, v6, v1
-; GFX9-IEEE-NEXT: v_mac_f32_e32 v6, v7, v3
-; GFX9-IEEE-NEXT: v_mad_f32 v0, -v0, v6, v1
-; GFX9-IEEE-NEXT: v_mul_f32_e32 v1, v4, v5
-; GFX9-IEEE-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX9-IEEE-NEXT: v_mad_f32 v3, -v2, v1, v4
-; GFX9-IEEE-NEXT: v_mac_f32_e32 v1, v3, v5
-; GFX9-IEEE-NEXT: v_mad_f32 v2, -v2, v1, v4
-; GFX9-IEEE-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v2, v5
-; GFX9-IEEE-NEXT: v_add_f32_e32 v0, v0, v6
-; GFX9-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v1, v4
+; GFX9-IEEE-NEXT: s_and_b32 s1, s1, 0xff800000
+; GFX9-IEEE-NEXT: v_add_f32_e32 v2, s1, v5
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v5, v0, v1
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v6, -v4, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v1
+; GFX9-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
+; GFX9-IEEE-NEXT: v_mul_f32_e64 v4, -v4, v5
+; GFX9-IEEE-NEXT: v_add_f32_e32 v0, v4, v0
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-IEEE-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-IEEE-NEXT: s_and_b32 s1, s1, 0xff800000
+; GFX9-IEEE-NEXT: v_add_f32_e32 v0, s1, v5
; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-IEEE-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX9-IEEE-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v0, s2, v2
-; GFX9-IEEE-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v1, s1, v2
-; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX9-IEEE-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v2, v3, s0
+; GFX9-IEEE-NEXT: v_readfirstlane_b32 s0, v1
+; GFX9-IEEE-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v0, v1, s2
+; GFX9-IEEE-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-IEEE-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-IEEE-NEXT: ; return to shader part epilog
;
; GFX9-FLUSH-LABEL: s_fdiv_v2f16:
; GFX9-FLUSH: ; %bb.0:
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
; GFX9-FLUSH-NEXT: s_lshr_b32 s2, s1, 16
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
-; GFX9-FLUSH-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-FLUSH-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-FLUSH-NEXT: s_lshr_b32 s3, s0, 16
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v1, v1
-; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, s0, v1, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -s1, v4, v2 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v4, v5, v1
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -s1, v4, v2 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v1, v5, v1
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v3, s0, v0, neg(0) op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v1, 0xff800000, v1
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v1, v1, v4
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -s1, v3, v2 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v2, v3, s0 op_sel_hi:[1,0,1]
; GFX9-FLUSH-NEXT: v_mac_f32_e32 v3, v4, v0
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v2, -s1, v3, v2 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v2, v0
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v2, v3, s0 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v4, v0
+; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v0, s1, v3
+; GFX9-FLUSH-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, s3, v1, neg(0) op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v3, v4, s3 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v4, v5, v1
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v3, v4, s3 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v1, v5, v1
+; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-FLUSH-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v1, s2, v2
-; GFX9-FLUSH-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, s1, v2
-; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v1, s1, v4
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v2, s0
; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v3, s3
+; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-FLUSH-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-FLUSH-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: s_fdiv_v2f16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshr_b32 s2, s1, 16
-; GFX10-NEXT: v_cvt_f32_f16_e32 v1, s1
-; GFX10-NEXT: v_cvt_f32_f16_e32 v0, s2
-; GFX10-NEXT: s_lshr_b32 s3, s0, 16
-; GFX10-NEXT: v_cvt_f32_f16_e32 v4, s0
-; GFX10-NEXT: v_cvt_f32_f16_e32 v5, s3
-; GFX10-NEXT: v_rcp_f32_e32 v3, v1
-; GFX10-NEXT: v_rcp_f32_e32 v2, v0
-; GFX10-NEXT: v_fma_mix_f32 v7, s0, v3, neg(0) op_sel_hi:[1,0,0]
-; GFX10-NEXT: v_fma_mix_f32 v6, s0, v2, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX10-NEXT: v_mad_f32 v9, -v1, v7, v4
-; GFX10-NEXT: v_mad_f32 v8, -v0, v6, v5
-; GFX10-NEXT: v_mac_f32_e32 v7, v9, v3
-; GFX10-NEXT: v_mac_f32_e32 v6, v8, v2
-; GFX10-NEXT: v_mad_f32 v1, -v1, v7, v4
-; GFX10-NEXT: v_mad_f32 v0, -v0, v6, v5
-; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX10-NEXT: v_and_b32_e32 v1, 0xff800000, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX10-NEXT: v_add_f32_e32 v1, v1, v7
-; GFX10-NEXT: v_add_f32_e32 v0, v0, v6
-; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_div_fixup_f16 v1, v1, s1, s0
-; GFX10-NEXT: v_div_fixup_f16 v0, v0, s2, s3
-; GFX10-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: ; return to shader part epilog
+; GFX10-IEEE-LABEL: s_fdiv_v2f16:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_lshr_b32 s2, s1, 16
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s1
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s2
+; GFX10-IEEE-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v0, v0
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v1, v1
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v2, s0, v0, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v3, s3, v1, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -s1, v2, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, -s2, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, s0, 1.0, v4 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, s3, 1.0, v5 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v4, v0
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v5, v5, v1
+; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v4, v2
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -s1, v2, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, -s2, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, s0, 1.0, v4 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, s3, 1.0, v5 op_sel_hi:[1,1,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v0, v4, v0
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v1, v5, v1
+; GFX10-IEEE-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-IEEE-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-IEEE-NEXT: s_and_b32 s4, s4, 0xff800000
+; GFX10-IEEE-NEXT: s_and_b32 s5, s5, 0xff800000
+; GFX10-IEEE-NEXT: v_add_f32_e32 v0, s4, v2
+; GFX10-IEEE-NEXT: v_add_f32_e32 v1, s5, v3
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v0, s1, s0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v1, s2, s3
+; GFX10-IEEE-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-IEEE-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-IEEE-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-IEEE-NEXT: ; return to shader part epilog
+;
+; GFX10-FLUSH-LABEL: s_fdiv_v2f16:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_lshr_b32 s2, s1, 16
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s2
+; GFX10-FLUSH-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, s0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v6, s3
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v2, v0
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v3, v1
+; GFX10-FLUSH-NEXT: v_fma_mix_f32 v5, s0, v2, neg(0) op_sel_hi:[1,0,0]
+; GFX10-FLUSH-NEXT: v_fma_mix_f32 v7, s3, v3, neg(0) op_sel_hi:[1,0,0]
+; GFX10-FLUSH-NEXT: v_mad_f32 v8, -v0, v5, v4
+; GFX10-FLUSH-NEXT: v_mad_f32 v9, -v1, v7, v6
+; GFX10-FLUSH-NEXT: v_mac_f32_e32 v5, v8, v2
+; GFX10-FLUSH-NEXT: v_mac_f32_e32 v7, v9, v3
+; GFX10-FLUSH-NEXT: v_mad_f32 v0, -v0, v5, v4
+; GFX10-FLUSH-NEXT: v_mad_f32 v1, -v1, v7, v6
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-FLUSH-NEXT: s_and_b32 s4, s4, 0xff800000
+; GFX10-FLUSH-NEXT: s_and_b32 s5, s5, 0xff800000
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v0, s4, v5
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v1, s5, v7
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v0, s1, s0
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v1, s2, s3
+; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-FLUSH-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-FLUSH-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fdiv_v2f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_lshr_b32 s2, s1, 16
-; GFX11-NEXT: v_cvt_f32_f16_e32 v1, s1
-; GFX11-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX11-NEXT: v_cvt_f32_f16_e32 v0, s1
+; GFX11-NEXT: v_cvt_f32_f16_e32 v1, s2
; GFX11-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-NEXT: v_rcp_f32_e32 v1, v1
; GFX11-NEXT: v_rcp_f32_e32 v0, v0
+; GFX11-NEXT: v_rcp_f32_e32 v1, v1
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_fma_mix_f32 v3, s0, v1, neg(0) op_sel_hi:[1,0,0]
-; GFX11-NEXT: v_fma_mix_f32 v2, s0, v0, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX11-NEXT: v_fma_mix_f32 v5, -s1, v3, s0 op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_fma_mix_f32 v4, -s1, v2, s0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_dual_fmac_f32 v3, v5, v1 :: v_dual_fmac_f32 v2, v4, v0
-; GFX11-NEXT: v_fma_mix_f32 v5, -s1, v3, s0 op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_fma_mix_f32 v4, -s1, v2, s0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_dual_mul_f32 v1, v5, v1 :: v_dual_mul_f32 v0, v4, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xff800000, v1
-; GFX11-NEXT: v_dual_add_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xff800000, v0
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX11-NEXT: v_fma_mix_f32 v2, s0, v0, neg(0) op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v3, s3, v1, neg(0) op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v4, -s1, v2, s0 op_sel_hi:[1,0,1]
+; GFX11-NEXT: v_fma_mix_f32 v5, -s2, v3, s3 op_sel_hi:[1,0,1]
+; GFX11-NEXT: v_dual_fmac_f32 v2, v4, v0 :: v_dual_fmac_f32 v3, v5, v1
+; GFX11-NEXT: v_fma_mix_f32 v4, -s1, v2, s0 op_sel_hi:[1,0,1]
+; GFX11-NEXT: v_fma_mix_f32 v5, -s2, v3, s3 op_sel_hi:[1,0,1]
+; GFX11-NEXT: v_dual_mul_f32 v0, v4, v0 :: v_dual_mul_f32 v1, v5, v1
+; GFX11-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-NEXT: v_readfirstlane_b32 s5, v1
+; GFX11-NEXT: s_and_b32 s4, s4, 0xff800000
+; GFX11-NEXT: s_and_b32 s5, s5, 0xff800000
+; GFX11-NEXT: v_dual_add_f32 v0, s4, v2 :: v_dual_add_f32 v1, s5, v3
; GFX11-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX11-NEXT: v_div_fixup_f16 v1, v1, s1, s0
-; GFX11-NEXT: v_div_fixup_f16 v0, v0, s2, s3
-; GFX11-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX11-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX11-NEXT: v_div_fixup_f16 v0, v0, s1, s0
+; GFX11-NEXT: v_div_fixup_f16 v1, v1, s2, s3
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%a = bitcast i32 %a.arg to <2 x half>
%b = bitcast i32 %b.arg to <2 x half>
@@ -3120,32 +3886,167 @@ define amdgpu_ps i32 @s_rsq_v2f16(i32 inreg %a.arg) {
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX9-LABEL: s_rsq_v2f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_lshr_b32 s1, s0, 16
-; GFX9-NEXT: v_rsq_f16_e32 v0, s1
-; GFX9-NEXT: v_rsq_f16_e32 v1, s0
-; GFX9-NEXT: v_pack_b32_f16 v0, -v1, -v0
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: s_rsq_v2f16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshr_b32 s1, s0, 16
-; GFX10-NEXT: v_rsq_f16_e32 v1, s0
-; GFX10-NEXT: v_rsq_f16_e32 v0, s1
-; GFX10-NEXT: v_pack_b32_f16 v0, -v1, -v0
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: ; return to shader part epilog
+; GFX9-IEEE-LABEL: s_rsq_v2f16:
+; GFX9-IEEE: ; %bb.0:
+; GFX9-IEEE-NEXT: v_sqrt_f16_e32 v0, s0
+; GFX9-IEEE-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-IEEE-NEXT: v_sqrt_f16_e32 v1, s0
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v2
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX9-IEEE-NEXT: v_fma_f32 v6, v2, v4, -1.0
+; GFX9-IEEE-NEXT: v_fma_f32 v6, v6, v4, -v4
+; GFX9-IEEE-NEXT: v_fma_f32 v7, v3, v5, -1.0
+; GFX9-IEEE-NEXT: v_fma_f32 v2, -v2, v6, -1.0
+; GFX9-IEEE-NEXT: v_fma_f32 v7, v7, v5, -v5
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX9-IEEE-NEXT: v_fma_f32 v3, -v3, v7, -1.0
+; GFX9-IEEE-NEXT: v_readfirstlane_b32 s0, v2
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v3, v5
+; GFX9-IEEE-NEXT: s_and_b32 s0, s0, 0xff800000
+; GFX9-IEEE-NEXT: v_readfirstlane_b32 s1, v2
+; GFX9-IEEE-NEXT: v_add_f32_e32 v2, s0, v6
+; GFX9-IEEE-NEXT: s_and_b32 s0, s1, 0xff800000
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, s0, v7
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
+; GFX9-IEEE-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v3, v1, -1.0
+; GFX9-IEEE-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-IEEE-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-IEEE-NEXT: ; return to shader part epilog
+;
+; GFX9-FLUSH-LABEL: s_rsq_v2f16:
+; GFX9-FLUSH: ; %bb.0:
+; GFX9-FLUSH-NEXT: v_sqrt_f16_e32 v0, s0
+; GFX9-FLUSH-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-FLUSH-NEXT: v_sqrt_f16_e32 v1, s0
+; GFX9-FLUSH-NEXT: v_mov_b32_e32 v4, -1.0
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, v0, v2, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_f32 v5, v5, v2, -v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, v1, v3, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v3, -v3
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v7, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v1, v6, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s0, v2
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v4, v3
+; GFX9-FLUSH-NEXT: s_and_b32 s0, s0, 0xff800000
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, s0, v5
+; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s0, v2
+; GFX9-FLUSH-NEXT: s_and_b32 s0, s0, 0xff800000
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, s0, v6
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
+; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, -1.0
+; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-FLUSH-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-FLUSH-NEXT: ; return to shader part epilog
+;
+; GFX10-IEEE-LABEL: s_rsq_v2f16:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10-IEEE-NEXT: v_sqrt_f16_e32 v0, s0
+; GFX10-IEEE-NEXT: v_sqrt_f16_e32 v1, s1
+; GFX10-IEEE-NEXT: v_mov_b32_e32 v4, -1.0
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, v0, v2, v4 op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v6, v1, v3, v4 op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_f32 v5, v5, v2, -v2
+; GFX10-IEEE-NEXT: v_fma_f32 v6, v6, v3, -v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v1, v6, v4 op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v7, v2
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX10-IEEE-NEXT: v_readfirstlane_b32 s0, v2
+; GFX10-IEEE-NEXT: v_readfirstlane_b32 s1, v3
+; GFX10-IEEE-NEXT: s_and_b32 s0, s0, 0xff800000
+; GFX10-IEEE-NEXT: s_and_b32 s1, s1, 0xff800000
+; GFX10-IEEE-NEXT: v_add_f32_e32 v2, s0, v5
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, s1, v6
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
+; GFX10-IEEE-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-IEEE-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-IEEE-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-IEEE-NEXT: ; return to shader part epilog
+;
+; GFX10-FLUSH-LABEL: s_rsq_v2f16:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10-FLUSH-NEXT: v_sqrt_f16_e32 v0, s0
+; GFX10-FLUSH-NEXT: v_sqrt_f16_e32 v1, s1
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, v2, v4, -1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, v3, v5, -1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, v6, v4, -v4
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, v7, v5, -v5
+; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v6, -1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, -1.0
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s0, v2
+; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s1, v3
+; GFX10-FLUSH-NEXT: s_and_b32 s0, s0, 0xff800000
+; GFX10-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, s0, v6
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, s1, v7
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
+; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-FLUSH-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-FLUSH-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_rsq_v2f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-NEXT: v_rsq_f16_e32 v1, s0
-; GFX11-NEXT: v_rsq_f16_e32 v0, s1
+; GFX11-NEXT: v_sqrt_f16_e32 v0, s0
+; GFX11-NEXT: v_sqrt_f16_e32 v1, s1
+; GFX11-NEXT: v_mov_b32_e32 v4, -1.0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_pack_b32_f16 v0, -v1, -v0
+; GFX11-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX11-NEXT: v_rcp_f32_e32 v2, v2
+; GFX11-NEXT: v_rcp_f32_e32 v3, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_fma_mix_f32 v5, v0, v2, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v6, v1, v3, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_f32 v5, v5, v2, -v2
+; GFX11-NEXT: v_fma_f32 v6, v6, v3, -v3
+; GFX11-NEXT: v_fma_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v4, -v1, v6, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_dual_mul_f32 v2, v7, v2 :: v_dual_mul_f32 v3, v4, v3
+; GFX11-NEXT: v_readfirstlane_b32 s0, v2
+; GFX11-NEXT: v_readfirstlane_b32 s1, v3
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff800000
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff800000
+; GFX11-NEXT: v_dual_add_f32 v2, s0, v5 :: v_dual_add_f32 v3, s1, v6
+; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
+; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
%a = bitcast i32 %a.arg to <2 x half>
%sqrt = call contract <2 x half> @llvm.sqrt.v2f16(<2 x half> %a)
@@ -4021,29 +4922,147 @@ define <2 x half> @v_rsq_v2f16(<2 x half> %a) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_rsq_v2f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rsq_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rsq_f16_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-IEEE-LABEL: v_rsq_v2f16:
+; GFX9-IEEE: ; %bb.0:
+; GFX9-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-IEEE-NEXT: v_sqrt_f16_e32 v1, v0
+; GFX9-IEEE-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v2
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX9-IEEE-NEXT: v_fma_f32 v6, -v2, v4, 1.0
+; GFX9-IEEE-NEXT: v_fma_f32 v7, -v3, v5, 1.0
+; GFX9-IEEE-NEXT: v_fma_f32 v6, v6, v4, v4
+; GFX9-IEEE-NEXT: v_fma_f32 v7, v7, v5, v5
+; GFX9-IEEE-NEXT: v_fma_f32 v2, -v2, v6, 1.0
+; GFX9-IEEE-NEXT: v_fma_f32 v3, -v3, v7, 1.0
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX9-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX9-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v3, v0, 1.0
+; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-IEEE-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_rsq_v2f16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rsq_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rsq_f16_e32 v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-FLUSH-LABEL: v_rsq_v2f16:
+; GFX9-FLUSH: ; %bb.0:
+; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-FLUSH-NEXT: v_sqrt_f16_e32 v1, v0
+; GFX9-FLUSH-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-FLUSH-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v1, v2, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v0, v3, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_f32 v5, v5, v2, v2
+; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v5, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v0, v6, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v7, v2
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, 1.0
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-IEEE-LABEL: v_rsq_v2f16:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-IEEE-NEXT: v_sqrt_f16_e32 v1, v0
+; GFX10-IEEE-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-IEEE-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, -v1, v2, v4 op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v6, -v0, v3, v4 op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_f32 v5, v5, v2, v2
+; GFX10-IEEE-NEXT: v_fma_f32 v6, v6, v3, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, -v1, v5, v4 op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v0, v6, v4 op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v7, v2
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v3, v0, 1.0
+; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-FLUSH-LABEL: v_rsq_v2f16:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-FLUSH-NEXT: v_sqrt_f16_e32 v1, v0
+; GFX10-FLUSH-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, -v2, v4, 1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, -v3, v5, 1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, v6, v4, v4
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, v7, v5, v5
+; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v6, 1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, 1.0
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, 1.0
+; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_rsq_v2f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-NEXT: v_rsq_f16_e32 v0, v0
-; GFX11-NEXT: v_rsq_f16_e32 v1, v1
+; GFX11-NEXT: v_sqrt_f16_e32 v0, v0
+; GFX11-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX11-NEXT: v_sqrt_f16_e32 v1, v1
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX11-NEXT: v_rcp_f32_e32 v2, v2
+; GFX11-NEXT: v_rcp_f32_e32 v3, v3
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_fma_mix_f32 v5, -v0, v2, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v6, -v1, v3, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_f32 v5, v5, v2, v2
+; GFX11-NEXT: v_fma_f32 v6, v6, v3, v3
+; GFX11-NEXT: v_fma_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v4, -v1, v6, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_dual_mul_f32 v2, v7, v2 :: v_dual_mul_f32 v3, v4, v3
+; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX11-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_and_b32 v3, 0xff800000, v3
+; GFX11-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
+; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%sqrt = call contract <2 x half> @llvm.sqrt.v2f16(<2 x half> %a)
@@ -4149,30 +5168,148 @@ define <2 x half> @v_neg_rsq_v2f16(<2 x half> %a) {
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_neg_rsq_v2f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rsq_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rsq_f16_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, -v0, -v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-IEEE-LABEL: v_neg_rsq_v2f16:
+; GFX9-IEEE: ; %bb.0:
+; GFX9-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-IEEE-NEXT: v_sqrt_f16_e32 v1, v0
+; GFX9-IEEE-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v2
+; GFX9-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX9-IEEE-NEXT: v_fma_f32 v6, v2, v4, -1.0
+; GFX9-IEEE-NEXT: v_fma_f32 v7, v3, v5, -1.0
+; GFX9-IEEE-NEXT: v_fma_f32 v6, v6, v4, -v4
+; GFX9-IEEE-NEXT: v_fma_f32 v7, v7, v5, -v5
+; GFX9-IEEE-NEXT: v_fma_f32 v2, -v2, v6, -1.0
+; GFX9-IEEE-NEXT: v_fma_f32 v3, -v3, v7, -1.0
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX9-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX9-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX9-IEEE-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
+; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
+; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-IEEE-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_neg_rsq_v2f16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rsq_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rsq_f16_e32 v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v0, -v0, -v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-FLUSH-LABEL: v_neg_rsq_v2f16:
+; GFX9-FLUSH: ; %bb.0:
+; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-FLUSH-NEXT: v_sqrt_f16_e32 v1, v0
+; GFX9-FLUSH-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-FLUSH-NEXT: v_mov_b32_e32 v4, -1.0
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, v1, v2, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, v0, v3, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_f32 v5, v5, v2, -v2
+; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v3, -v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v5, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v0, v6, v4 op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v7, v2
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-IEEE-LABEL: v_neg_rsq_v2f16:
+; GFX10-IEEE: ; %bb.0:
+; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-IEEE-NEXT: v_sqrt_f16_e32 v1, v0
+; GFX10-IEEE-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-IEEE-NEXT: v_mov_b32_e32 v4, -1.0
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_rcp_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v5, v1, v2, v4 op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v6, v0, v3, v4 op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_f32 v5, v5, v2, -v2
+; GFX10-IEEE-NEXT: v_fma_f32 v6, v6, v3, -v3
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, -v1, v5, v4 op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v0, v6, v4 op_sel_hi:[1,0,0]
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v7, v2
+; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
+; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-FLUSH-LABEL: v_neg_rsq_v2f16:
+; GFX10-FLUSH: ; %bb.0:
+; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-FLUSH-NEXT: v_sqrt_f16_e32 v1, v0
+; GFX10-FLUSH-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, v2, v4, -1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, v3, v5, -1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v6, v6, v4, -v4
+; GFX10-FLUSH-NEXT: v_mad_f32 v7, v7, v5, -v5
+; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v6, -1.0
+; GFX10-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, -1.0
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
+; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
+; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_neg_rsq_v2f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-NEXT: v_rsq_f16_e32 v0, v0
-; GFX11-NEXT: v_rsq_f16_e32 v1, v1
+; GFX11-NEXT: v_sqrt_f16_e32 v0, v0
+; GFX11-NEXT: v_mov_b32_e32 v4, -1.0
+; GFX11-NEXT: v_sqrt_f16_e32 v1, v1
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_pack_b32_f16 v0, -v0, -v1
+; GFX11-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX11-NEXT: v_rcp_f32_e32 v2, v2
+; GFX11-NEXT: v_rcp_f32_e32 v3, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_fma_mix_f32 v5, v0, v2, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v6, v1, v3, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_f32 v5, v5, v2, -v2
+; GFX11-NEXT: v_fma_f32 v6, v6, v3, -v3
+; GFX11-NEXT: v_fma_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_fma_mix_f32 v4, -v1, v6, v4 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_dual_mul_f32 v2, v7, v2 :: v_dual_mul_f32 v3, v4, v3
+; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX11-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_and_b32 v3, 0xff800000, v3
+; GFX11-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
+; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
+; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%sqrt = call contract <2 x half> @llvm.sqrt.v2f16(<2 x half> %a)
%fdiv = fdiv contract <2 x half> <half -1.0, half -1.0>, %sqrt
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
index 683e7965a74fe..072eb60352c6f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=GFX6 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
index fc1aecdf20d1c..ecc137621aeab 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-mesa3d -mcpu=gfx803 < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1170 -mattr=-real-true16 < %s | FileCheck -check-prefix=GFX1170 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1170 -mattr=-real-true16 < %s | FileCheck -check-prefix=GFX1170 %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
define float @test_min_max_ValK0_K1_f32(float %a) #0 {
; GFX8-LABEL: test_min_max_ValK0_K1_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
index aac0c33317f56..7d35eb2e8fa38 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
@@ -239,37 +239,38 @@ define <2 x half> @v_pow_v2f16(<2 x half> %x, <2 x half> %y) {
; GFX9-LABEL: v_pow_v2f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_log_f16_e32 v2, v0
+; GFX9-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX9-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX9-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9-NEXT: v_log_f32_e32 v2, v2
-; GFX9-NEXT: v_log_f32_e32 v0, v0
-; GFX9-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
-; GFX9-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX9-NEXT: v_exp_f32_e32 v1, v2
-; GFX9-NEXT: v_exp_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
+; GFX9-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_cvt_f16_f32_e32 v1, v2
; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-NEXT: v_exp_f16_e32 v1, v1
+; GFX9-NEXT: v_exp_f16_e32 v0, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_pow_v2f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_log_f16_e32 v2, v0
+; GFX10-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX10-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX10-NEXT: v_log_f32_e32 v2, v2
-; GFX10-NEXT: v_log_f32_e32 v0, v0
-; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
-; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX10-NEXT: v_exp_f32_e32 v1, v2
-; GFX10-NEXT: v_exp_f32_e32 v0, v0
-; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-NEXT: v_exp_f16_e32 v1, v2
+; GFX10-NEXT: v_exp_f16_e32 v0, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_v2f16:
@@ -295,27 +296,27 @@ define <2 x half> @v_pow_v2f16(<2 x half> %x, <2 x half> %y) {
; GFX11-FAKE16-LABEL: v_pow_v2f16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_log_f16_e32 v2, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX11-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_log_f16_e32 v0, v0
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX11-FAKE16-NEXT: v_log_f32_e32 v2, v2
-; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-FAKE16-NEXT: v_mul_dx9_zero_f32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_mul_dx9_zero_f32_e32 v2, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX11-FAKE16-NEXT: v_exp_f32_e32 v1, v2
; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mul_dx9_zero_f32 v1, v2, v1 :: v_dual_mul_dx9_zero_f32 v0, v0, v3
; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_exp_f16_e32 v1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_exp_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%pow = call <2 x half> @llvm.pow.v2f16(<2 x half> %x, <2 x half> %y)
ret <2 x half> %pow
@@ -365,37 +366,40 @@ define <2 x half> @v_pow_v2f16_fneg_lhs(<2 x half> %x, <2 x half> %y) {
; GFX9-LABEL: v_pow_v2f16_fneg_lhs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_f16_sdwa v2, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; GFX9-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9-NEXT: v_log_f32_e32 v2, v2
-; GFX9-NEXT: v_log_f32_e32 v0, v0
-; GFX9-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
-; GFX9-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX9-NEXT: v_exp_f32_e32 v1, v2
-; GFX9-NEXT: v_exp_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_log_f16_e32 v2, v0
+; GFX9-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX9-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX9-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
+; GFX9-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_cvt_f16_f32_e32 v1, v2
; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-NEXT: v_exp_f16_e32 v1, v1
+; GFX9-NEXT: v_exp_f16_e32 v0, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_pow_v2f16_fneg_lhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_f16_sdwa v2, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; GFX10-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX10-NEXT: v_log_f32_e32 v2, v2
-; GFX10-NEXT: v_log_f32_e32 v0, v0
-; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
-; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX10-NEXT: v_exp_f32_e32 v1, v2
-; GFX10-NEXT: v_exp_f32_e32 v0, v0
-; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_log_f16_e32 v2, v0
+; GFX10-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-NEXT: v_exp_f16_e32 v1, v2
+; GFX10-NEXT: v_exp_f16_e32 v0, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_v2f16_fneg_lhs:
@@ -423,27 +427,29 @@ define <2 x half> @v_pow_v2f16_fneg_lhs(<2 x half> %x, <2 x half> %y) {
; GFX11-FAKE16-LABEL: v_pow_v2f16_fneg_lhs:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e64 v2, -v2
-; GFX11-FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_log_f16_e32 v2, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX11-FAKE16-NEXT: v_log_f32_e32 v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_log_f16_e32 v0, v0
; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-FAKE16-NEXT: v_mul_dx9_zero_f32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_mul_dx9_zero_f32_e32 v2, v3, v2
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_dual_mul_dx9_zero_f32 v1, v2, v1 :: v_dual_mul_dx9_zero_f32 v0, v0, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX11-FAKE16-NEXT: v_exp_f32_e32 v1, v2
+; GFX11-FAKE16-NEXT: v_exp_f16_e32 v1, v1
+; GFX11-FAKE16-NEXT: v_exp_f16_e32 v0, v0
; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%x.fneg = fneg <2 x half> %x
%pow = call <2 x half> @llvm.pow.v2f16(<2 x half> %x.fneg, <2 x half> %y)
@@ -494,37 +500,40 @@ define <2 x half> @v_pow_v2f16_fneg_rhs(<2 x half> %x, <2 x half> %y) {
; GFX9-LABEL: v_pow_v2f16_fneg_rhs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_log_f16_e32 v2, v0
+; GFX9-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX9-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX9-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX9-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX9-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_f16_sdwa v3, -v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_cvt_f32_f16_e64 v1, -v1
-; GFX9-NEXT: v_log_f32_e32 v2, v2
-; GFX9-NEXT: v_log_f32_e32 v0, v0
-; GFX9-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
-; GFX9-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX9-NEXT: v_exp_f32_e32 v1, v2
-; GFX9-NEXT: v_exp_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
+; GFX9-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-NEXT: v_exp_f16_e32 v1, v2
+; GFX9-NEXT: v_exp_f16_e32 v0, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_pow_v2f16_fneg_rhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_log_f16_e32 v2, v0
+; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX10-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX10-NEXT: v_cvt_f32_f16_sdwa v3, -v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_cvt_f32_f16_e64 v1, -v1
-; GFX10-NEXT: v_log_f32_e32 v2, v2
-; GFX10-NEXT: v_log_f32_e32 v0, v0
-; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
-; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX10-NEXT: v_exp_f32_e32 v1, v2
-; GFX10-NEXT: v_exp_f32_e32 v0, v0
-; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-NEXT: v_exp_f16_e32 v1, v2
+; GFX10-NEXT: v_exp_f16_e32 v0, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_v2f16_fneg_rhs:
@@ -552,27 +561,28 @@ define <2 x half> @v_pow_v2f16_fneg_rhs(<2 x half> %x, <2 x half> %y) {
; GFX11-FAKE16-LABEL: v_pow_v2f16_fneg_rhs:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e64 v1, -v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX11-FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e64 v3, -v3
-; GFX11-FAKE16-NEXT: v_log_f32_e32 v2, v2
-; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-FAKE16-NEXT: v_mul_dx9_zero_f32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_mul_dx9_zero_f32_e32 v2, v3, v2
+; GFX11-FAKE16-NEXT: v_log_f16_e32 v2, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX11-FAKE16-NEXT: v_exp_f32_e32 v1, v2
+; GFX11-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mul_dx9_zero_f32 v1, v2, v1 :: v_dual_mul_dx9_zero_f32 v0, v0, v3
; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_exp_f16_e32 v1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_exp_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%y.fneg = fneg <2 x half> %y
%pow = call <2 x half> @llvm.pow.v2f16(<2 x half> %x, <2 x half> %y.fneg)
@@ -624,37 +634,42 @@ define <2 x half> @v_pow_v2f16_fneg_lhs_rhs(<2 x half> %x, <2 x half> %y) {
; GFX9-LABEL: v_pow_v2f16_fneg_lhs_rhs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_f16_sdwa v2, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; GFX9-NEXT: v_cvt_f32_f16_sdwa v3, -v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_cvt_f32_f16_e64 v1, -v1
-; GFX9-NEXT: v_log_f32_e32 v2, v2
-; GFX9-NEXT: v_log_f32_e32 v0, v0
-; GFX9-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
-; GFX9-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX9-NEXT: v_exp_f32_e32 v1, v2
-; GFX9-NEXT: v_exp_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_log_f16_e32 v2, v0
+; GFX9-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX9-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX9-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX9-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
+; GFX9-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-NEXT: v_exp_f16_e32 v1, v2
+; GFX9-NEXT: v_exp_f16_e32 v0, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_pow_v2f16_fneg_lhs_rhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_f16_sdwa v2, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; GFX10-NEXT: v_cvt_f32_f16_sdwa v3, -v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_cvt_f32_f16_e64 v1, -v1
-; GFX10-NEXT: v_log_f32_e32 v2, v2
-; GFX10-NEXT: v_log_f32_e32 v0, v0
-; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
-; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX10-NEXT: v_exp_f32_e32 v1, v2
-; GFX10-NEXT: v_exp_f32_e32 v0, v0
-; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX10-NEXT: v_log_f16_e32 v2, v0
+; GFX10-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX10-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-NEXT: v_exp_f16_e32 v1, v2
+; GFX10-NEXT: v_exp_f16_e32 v0, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_v2f16_fneg_lhs_rhs:
@@ -684,27 +699,30 @@ define <2 x half> @v_pow_v2f16_fneg_lhs_rhs(<2 x half> %x, <2 x half> %y) {
; GFX11-FAKE16-LABEL: v_pow_v2f16_fneg_lhs_rhs:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_log_f16_e32 v2, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e64 v1, -v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e64 v2, -v2
-; GFX11-FAKE16-NEXT: v_log_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e64 v3, -v3
-; GFX11-FAKE16-NEXT: v_log_f32_e32 v2, v2
-; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-FAKE16-NEXT: v_mul_dx9_zero_f32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_mul_dx9_zero_f32_e32 v2, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX11-FAKE16-NEXT: v_exp_f32_e32 v1, v2
+; GFX11-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mul_dx9_zero_f32 v1, v2, v1 :: v_dual_mul_dx9_zero_f32 v0, v0, v3
; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_exp_f16_e32 v1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_exp_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%x.fneg = fneg <2 x half> %x
%y.fneg = fneg <2 x half> %y
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
index d41e213da1c74..6f2aa60ce9fe4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GPRIDX %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GPRIDX %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GFX8 %s
define amdgpu_ps <8 x i32> @dyn_insertelement_v8i32_s_s_s(<8 x i32> inreg %vec, i32 inreg %val, i32 inreg %idx) {
; GPRIDX-LABEL: dyn_insertelement_v8i32_s_s_s:
@@ -8544,7 +8544,10 @@ define void @insert_very_small_from_very_large(<32 x i16> %L3, ptr %ptr) {
; GPRIDX-LABEL: insert_very_small_from_very_large:
; GPRIDX: ; %bb.0: ; %bb
; GPRIDX-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GPRIDX-NEXT: v_and_b32_e32 v0, 2, v0
+; GPRIDX-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GPRIDX-NEXT: v_and_b32_e32 v0, 1, v0
+; GPRIDX-NEXT: v_lshlrev_b16_e32 v0, 1, v0
+; GPRIDX-NEXT: v_and_b32_e32 v0, 3, v0
; GPRIDX-NEXT: flat_store_byte v[16:17], v0
; GPRIDX-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GPRIDX-NEXT: s_setpc_b64 s[30:31]
@@ -8552,7 +8555,10 @@ define void @insert_very_small_from_very_large(<32 x i16> %L3, ptr %ptr) {
; GFX10-LABEL: insert_very_small_from_very_large:
; GFX10: ; %bb.0: ; %bb
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v0, 2, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX10-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT: v_lshlrev_b16 v0, 1, v0
+; GFX10-NEXT: v_and_b32_e32 v0, 3, v0
; GFX10-NEXT: flat_store_byte v[16:17], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -8571,7 +8577,10 @@ define void @insert_very_small_from_very_large(<32 x i16> %L3, ptr %ptr) {
; GFX8-LABEL: insert_very_small_from_very_large:
; GFX8: ; %bb.0: ; %bb
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v0, 2, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX8-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT: v_lshlrev_b16_e32 v0, 1, v0
+; GFX8-NEXT: v_and_b32_e32 v0, 3, v0
; GFX8-NEXT: flat_store_byte v[16:17], v0
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-abs.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-abs.mir
index 0774cd14a9fec..59a3293de36f4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-abs.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-abs.mir
@@ -295,14 +295,15 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; SI-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; SI-NEXT: [[ABS:%[0-9]+]]:_(s32) = G_ABS [[SEXT_INREG]]
; SI-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; SI-NEXT: [[ABS1:%[0-9]+]]:_(s32) = G_ABS [[SEXT_INREG1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ABS1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ABS1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ABS]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -311,16 +312,17 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[ABS:%[0-9]+]]:_(s16) = G_ABS [[TRUNC]]
; VI-NEXT: [[ABS1:%[0-9]+]]:_(s16) = G_ABS [[TRUNC1]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ABS]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ABS1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -372,11 +374,11 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX9-NEXT: [[ABS:%[0-9]+]]:_(<2 x s16>) = G_ABS [[BUILD_VECTOR]]
; GFX9-NEXT: [[ABS1:%[0-9]+]]:_(s16) = G_ABS [[DEF]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[ABS]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[ABS]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ABS1]](s16)
- ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[ANYEXT]](s32)
+ ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[ANYEXT]](s32)
; GFX9-NEXT: S_NOP 0, implicit [[BUILD_VECTOR1]](<3 x s32>)
%0:_(<3 x s16>) = G_IMPLICIT_DEF
%1:_(<3 x s16>) = G_ABS %0
@@ -395,11 +397,11 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; SI-NEXT: [[ABS:%[0-9]+]]:_(s32) = G_ABS [[SEXT_INREG]]
; SI-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
@@ -408,10 +410,11 @@ body: |
; SI-NEXT: [[ABS2:%[0-9]+]]:_(s32) = G_ABS [[SEXT_INREG2]]
; SI-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR1]], 16
; SI-NEXT: [[ABS3:%[0-9]+]]:_(s32) = G_ABS [[SEXT_INREG3]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ABS1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ABS1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ABS]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ABS3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ABS3]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ABS2]], [[SHL1]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -422,27 +425,28 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[ABS:%[0-9]+]]:_(s16) = G_ABS [[TRUNC]]
; VI-NEXT: [[ABS1:%[0-9]+]]:_(s16) = G_ABS [[TRUNC1]]
; VI-NEXT: [[ABS2:%[0-9]+]]:_(s16) = G_ABS [[TRUNC2]]
; VI-NEXT: [[ABS3:%[0-9]+]]:_(s16) = G_ABS [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ABS]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ABS1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ABS2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ABS3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir
index f94428110c600..0e1ba2d9923be 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir
@@ -144,17 +144,18 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[BITCAST]], [[BITCAST1]]
; GFX6-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR1]]
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C1]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -164,20 +165,21 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[TRUNC2]]
; GFX8-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[TRUNC3]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -259,13 +261,13 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC5]](s16), [[DEF]](s16)
; GFX9-NEXT: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
; GFX9-NEXT: [[ADD1:%[0-9]+]]:_(<2 x s16>) = G_ADD [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[ADD]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[ADD1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[ADD]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[ADD1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: S_ENDPGM 0, implicit [[TRUNC6]](s16), implicit [[TRUNC7]](s16), implicit [[TRUNC8]](s16)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
@@ -298,16 +300,16 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; GFX6-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[BITCAST]], [[BITCAST2]]
; GFX6-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR2]]
; GFX6-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[BITCAST1]], [[BITCAST3]]
@@ -315,12 +317,13 @@ body: |
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C1]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C1]]
; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD3]], [[C1]]
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -332,36 +335,37 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX8-NEXT: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[TRUNC4]]
; GFX8-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[TRUNC5]]
; GFX8-NEXT: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[TRUNC6]]
; GFX8-NEXT: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[TRUNC3]], [[TRUNC7]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
index aa198a5c3f4d7..702cf1257b6f2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
@@ -514,9 +514,10 @@ body: |
; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; CHECK-NEXT: [[AND:%[0-9]+]]:_(<4 x s16>) = G_AND [[CONCAT_VECTORS]], [[CONCAT_VECTORS1]]
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[AND]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C2]](i32)
+ ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL4]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
@@ -533,15 +534,15 @@ body: |
; CHECK-NEXT: [[CONCAT_VECTORS3:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST8]](<2 x s16>), [[BITCAST9]](<2 x s16>)
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<4 x s16>) = G_AND [[CONCAT_VECTORS2]], [[CONCAT_VECTORS3]]
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[AND1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST10]], [[C1]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST4]], [[C2]]
+ ; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C2]](i32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST4]], [[C3]]
; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
; CHECK-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL8]]
; CHECK-NEXT: [[BITCAST11:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR8]](i32)
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST5]], [[C2]]
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[BITCAST10]], [[C2]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST5]], [[C3]]
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[BITCAST10]], [[C3]]
; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND4]], [[C1]](s32)
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[AND3]], [[SHL9]]
; CHECK-NEXT: [[BITCAST12:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR9]](i32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-anyext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-anyext.mir
index d2c496e5a349b..44a0b073f6673 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-anyext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-anyext.mir
@@ -106,10 +106,10 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(<2 x s32>) = G_ANYEXT %0
@@ -127,11 +127,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[BITCAST1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[BITCAST1]](i32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
%1:_(<3 x s16>) = G_EXTRACT %0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
index 65bb2ff1cb2cd..e4000c455c28f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
@@ -700,20 +700,21 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; SI-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG]], [[AND]](s32)
; SI-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
- ; SI-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG1]], [[LSHR1]](s32)
+ ; SI-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG1]], [[LSHR1]](i32)
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C1]]
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -723,20 +724,21 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC2]](s16)
; VI-NEXT: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC1]], [[TRUNC3]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -765,9 +767,9 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
; SI-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; SI-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG]], [[UV]](s32)
@@ -776,7 +778,8 @@ body: |
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -786,11 +789,11 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
; VI-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC2]](s16)
@@ -798,7 +801,8 @@ body: |
; VI-NEXT: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC1]], [[TRUNC3]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -808,11 +812,11 @@ body: |
; GFX9PLUS-NEXT: {{ $}}
; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
- ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9PLUS-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
; GFX9PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
; GFX9PLUS-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC2]](s16)
@@ -837,35 +841,36 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; SI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
; SI-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; SI-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG]], [[AND]](s32)
; SI-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
- ; SI-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG1]], [[LSHR1]](s32)
+ ; SI-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG1]], [[LSHR1]](i32)
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; SI-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG2]], [[AND1]](s32)
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C1]]
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL2]]
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -876,35 +881,36 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; VI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; VI-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC3]](s16)
; VI-NEXT: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC1]], [[TRUNC4]](s16)
; VI-NEXT: [[ASHR2:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC2]], [[TRUNC5]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR2]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL2]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL2]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -915,20 +921,20 @@ body: |
; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; GFX9PLUS-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX9PLUS-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
; GFX9PLUS-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; GFX9PLUS-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9PLUS-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9PLUS-NEXT: [[UV9:%[0-9]+]]:_(<2 x s16>), [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
; GFX9PLUS-NEXT: [[ASHR:%[0-9]+]]:_(<2 x s16>) = G_ASHR [[UV3]], [[UV9]](<2 x s16>)
; GFX9PLUS-NEXT: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC1]](s16)
- ; GFX9PLUS-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[ASHR]](<2 x s16>)
- ; GFX9PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9PLUS-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9PLUS-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[ASHR]](<2 x s16>)
+ ; GFX9PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9PLUS-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9PLUS-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
; GFX9PLUS-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[ASHR1]](s16), [[DEF]](s16)
@@ -958,35 +964,36 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
; SI-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; SI-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG]], [[AND]](s32)
; SI-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
- ; SI-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG1]], [[LSHR2]](s32)
+ ; SI-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG1]], [[LSHR2]](i32)
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; SI-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG2]], [[AND1]](s32)
; SI-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR1]], 16
- ; SI-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG3]], [[LSHR3]](s32)
+ ; SI-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG3]], [[LSHR3]](i32)
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C1]]
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C1]]
; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C1]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -998,36 +1005,37 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC4]](s16)
; VI-NEXT: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC1]], [[TRUNC5]](s16)
; VI-NEXT: [[ASHR2:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC2]], [[TRUNC6]](s16)
; VI-NEXT: [[ASHR3:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC3]], [[TRUNC7]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
index 540ece5599369..5a6cc6f98abd8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
@@ -557,23 +557,23 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C2]](i32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s16) = COPY [[TRUNC]](s16)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[COPY1]]
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s16) = COPY [[TRUNC1]](s16)
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[COPY2]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s16) = COPY [[TRUNC2]](s16)
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[COPY3]]
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s16) = COPY [[TRUNC3]](s16)
; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[TRUNC3]], [[COPY4]]
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ADD]](s16)
@@ -646,43 +646,43 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C3]](s32)
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
- ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C4]](s32)
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C5]](s32)
- ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
- ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C6]](s32)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 12
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C2]](i32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C3]](i32)
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 20
+ ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C4]](i32)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C5]](i32)
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 28
+ ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C6]](i32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s16) = COPY [[TRUNC]](s16)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[COPY1]]
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s16) = COPY [[TRUNC1]](s16)
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[COPY2]]
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s16) = COPY [[TRUNC2]](s16)
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[COPY3]]
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s16) = COPY [[TRUNC3]](s16)
; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[TRUNC3]], [[COPY4]]
- ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s16) = COPY [[TRUNC4]](s16)
; CHECK-NEXT: [[ADD4:%[0-9]+]]:_(s16) = G_ADD [[TRUNC4]], [[COPY5]]
- ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
+ ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s16) = COPY [[TRUNC5]](s16)
; CHECK-NEXT: [[ADD5:%[0-9]+]]:_(s16) = G_ADD [[TRUNC5]], [[COPY6]]
- ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; CHECK-NEXT: [[COPY7:%[0-9]+]]:_(s16) = COPY [[TRUNC6]](s16)
; CHECK-NEXT: [[ADD6:%[0-9]+]]:_(s16) = G_ADD [[TRUNC6]], [[COPY7]]
- ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](s32)
+ ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](i32)
; CHECK-NEXT: [[COPY8:%[0-9]+]]:_(s16) = COPY [[TRUNC7]](s16)
; CHECK-NEXT: [[ADD7:%[0-9]+]]:_(s16) = G_ADD [[TRUNC7]], [[COPY8]]
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ADD]](s16)
@@ -804,35 +804,36 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<4 x s16>) = G_BITCAST [[COPY]](<2 x s32>)
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[BITCAST]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[TRUNC4]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[TRUNC5]]
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[TRUNC6]]
; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[TRUNC3]], [[TRUNC7]]
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
@@ -1152,31 +1153,31 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<16 x s16>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>), [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<16 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV6]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST7]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s4) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s4) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s4) = G_TRUNC [[BITCAST1]](s32)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s4) = G_TRUNC [[LSHR1]](s32)
- ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s4) = G_TRUNC [[BITCAST2]](s32)
- ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s4) = G_TRUNC [[LSHR2]](s32)
- ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s4) = G_TRUNC [[BITCAST3]](s32)
- ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s4) = G_TRUNC [[LSHR3]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[UV6]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s4) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s4) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s4) = G_TRUNC [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s4) = G_TRUNC [[LSHR1]](i32)
+ ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s4) = G_TRUNC [[BITCAST2]](i32)
+ ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s4) = G_TRUNC [[LSHR2]](i32)
+ ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s4) = G_TRUNC [[BITCAST3]](i32)
+ ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s4) = G_TRUNC [[LSHR3]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
@@ -1192,19 +1193,20 @@ body: |
; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND4]], [[C]](s32)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND4]], [[C5]](s32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[SHL3]]
; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C1]]
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C5]](s32)
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C6]](s32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL4]]
; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
- ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND6]], [[C6]](s32)
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND6]], [[C7]](s32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[SHL5]]
; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C1]]
- ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C7]](s32)
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C8]](s32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[OR5]], [[SHL6]]
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s4) = COPY [[TRUNC]](s4)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s4) = COPY [[TRUNC1]](s4)
@@ -1225,16 +1227,16 @@ body: |
; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C4]](s32)
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[OR8]], [[SHL9]]
; CHECK-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[BITCAST6]], [[C1]]
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[AND12]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[AND12]], [[C5]](s32)
; CHECK-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[OR9]], [[SHL10]]
; CHECK-NEXT: [[AND13:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C1]]
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[AND13]], [[C5]](s32)
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[AND13]], [[C6]](s32)
; CHECK-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[OR10]], [[SHL11]]
; CHECK-NEXT: [[AND14:%[0-9]+]]:_(s32) = G_AND [[BITCAST7]], [[C1]]
- ; CHECK-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[AND14]], [[C6]](s32)
+ ; CHECK-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[AND14]], [[C7]](s32)
; CHECK-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[OR11]], [[SHL12]]
; CHECK-NEXT: [[AND15:%[0-9]+]]:_(s32) = G_AND [[LSHR7]], [[C1]]
- ; CHECK-NEXT: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[AND15]], [[C7]](s32)
+ ; CHECK-NEXT: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[AND15]], [[C8]](s32)
; CHECK-NEXT: [[OR13:%[0-9]+]]:_(s32) = G_OR [[OR12]], [[SHL13]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR6]](s32), [[OR13]](s32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<2 x s32>)
@@ -1938,23 +1940,23 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<8 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[C1]](s16)
; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C1]](s16)
@@ -2354,23 +2356,23 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<12 x s16>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<12 x s16>) = COPY $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<12 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C1]]
@@ -2391,30 +2393,31 @@ body: |
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[SHL3]]
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C3]](s32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR4]](s32), [[OR5]](s32)
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>), [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<12 x s16>)
- ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV6]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC12:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST6]](s32)
- ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC13:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](s32)
- ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC14:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST7]](s32)
- ; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST7]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC15:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](s32)
+ ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[UV6]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC12:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST6]](i32)
+ ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC13:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](i32)
+ ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC14:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST7]](i32)
+ ; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC15:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](i32)
; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C1]]
; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s16) = G_AND [[TRUNC9]], [[C1]]
; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[AND9]], [[C2]](s16)
@@ -2433,29 +2436,29 @@ body: |
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(s16) = G_OR [[AND14]], [[SHL9]]
; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C3]](s32)
; CHECK-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[ZEXT4]], [[SHL10]]
; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR8]](s16)
; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR9]](s16)
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C3]](s32)
; CHECK-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[ZEXT6]], [[SHL11]]
; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR10]](s32), [[OR11]](s32)
- ; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC16:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST8]](s32)
- ; CHECK-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST8]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC17:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR8]](s32)
- ; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC18:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST9]](s32)
- ; CHECK-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST9]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC19:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR9]](s32)
- ; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC20:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST10]](s32)
- ; CHECK-NEXT: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST10]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC21:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR10]](s32)
- ; CHECK-NEXT: [[BITCAST11:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC22:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST11]](s32)
- ; CHECK-NEXT: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST11]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC23:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR11]](s32)
+ ; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(i32) = G_BITCAST [[UV8]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC16:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST8]](i32)
+ ; CHECK-NEXT: [[LSHR8:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST8]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC17:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR8]](i32)
+ ; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(i32) = G_BITCAST [[UV9]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC18:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST9]](i32)
+ ; CHECK-NEXT: [[LSHR9:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST9]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC19:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR9]](i32)
+ ; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[UV10]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC20:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST10]](i32)
+ ; CHECK-NEXT: [[LSHR10:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC21:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR10]](i32)
+ ; CHECK-NEXT: [[BITCAST11:%[0-9]+]]:_(i32) = G_BITCAST [[UV11]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC22:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST11]](i32)
+ ; CHECK-NEXT: [[LSHR11:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST11]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC23:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR11]](i32)
; CHECK-NEXT: [[AND16:%[0-9]+]]:_(s16) = G_AND [[TRUNC16]], [[C1]]
; CHECK-NEXT: [[AND17:%[0-9]+]]:_(s16) = G_AND [[TRUNC17]], [[C1]]
; CHECK-NEXT: [[SHL12:%[0-9]+]]:_(s16) = G_SHL [[AND17]], [[C2]](s16)
@@ -2474,11 +2477,11 @@ body: |
; CHECK-NEXT: [[OR15:%[0-9]+]]:_(s16) = G_OR [[AND22]], [[SHL15]]
; CHECK-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR12]](s16)
; CHECK-NEXT: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[OR13]](s16)
- ; CHECK-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C3]](s32)
; CHECK-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[ZEXT8]], [[SHL16]]
; CHECK-NEXT: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[OR14]](s16)
; CHECK-NEXT: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[OR15]](s16)
- ; CHECK-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[ZEXT11]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[ZEXT11]], [[C3]](s32)
; CHECK-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[ZEXT10]], [[SHL17]]
; CHECK-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR16]](s32), [[OR17]](s32)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64)
@@ -2502,15 +2505,15 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[C1]](s16)
; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C1]](s16)
@@ -2526,22 +2529,23 @@ body: |
; CHECK-NEXT: [[ADD7:%[0-9]+]]:_(s16) = G_ADD [[LSHR5]], [[LSHR5]]
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[ADD4]](s16)
; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[ADD5]](s16)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C2]](s32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL2]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[ADD6]](s16)
; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[ADD7]](s16)
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C2]](s32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT6]], [[SHL3]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -2564,25 +2568,25 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<8 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
- ; CHECK-NEXT: [[TRUNC8:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[TRUNC9:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
+ ; CHECK-NEXT: [[TRUNC8:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[TRUNC9:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C1]]
@@ -2591,8 +2595,8 @@ body: |
; CHECK-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s8) = COPY [[TRUNC8]](s8)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s8) = COPY [[TRUNC9]](s8)
- ; CHECK-NEXT: [[TRUNC10:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST1]](s32)
- ; CHECK-NEXT: [[TRUNC11:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR1]](s32)
+ ; CHECK-NEXT: [[TRUNC10:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST1]](i32)
+ ; CHECK-NEXT: [[TRUNC11:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR1]](i32)
; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C1]]
; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C1]]
; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C2]](s16)
@@ -2601,8 +2605,8 @@ body: |
; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s8) = COPY [[TRUNC9]](s8)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s8) = COPY [[TRUNC10]](s8)
; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s8) = COPY [[TRUNC11]](s8)
- ; CHECK-NEXT: [[TRUNC12:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST2]](s32)
- ; CHECK-NEXT: [[TRUNC13:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR2]](s32)
+ ; CHECK-NEXT: [[TRUNC12:%[0-9]+]]:_(s8) = G_TRUNC [[BITCAST2]](i32)
+ ; CHECK-NEXT: [[TRUNC13:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR2]](i32)
; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C1]]
; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C1]]
; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
@@ -2623,12 +2627,13 @@ body: |
; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[OR3]], [[OR3]]
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C3]](s32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitreverse.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitreverse.mir
index 770004e63f855..1674bd66ba79a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitreverse.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitreverse.mir
@@ -91,14 +91,15 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[BITREVERSE:%[0-9]+]]:_(s32) = G_BITREVERSE [[BITCAST]]
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITREVERSE]], [[C]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITREVERSE]], [[C1]](s32)
; CHECK-NEXT: [[BITREVERSE1:%[0-9]+]]:_(s32) = G_BITREVERSE [[LSHR]]
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITREVERSE1]], [[C]](s32)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C]](s32)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITREVERSE1]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C1]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bswap.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bswap.mir
index 5f63bae1213d3..a4d06988c4f3b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bswap.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bswap.mir
@@ -73,9 +73,9 @@ body: |
; GFX7: liveins: $vgpr0
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX7-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
@@ -94,7 +94,8 @@ body: |
; GFX7-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC3]], [[TRUNC2]]
; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX7-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX7-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
; GFX7-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX7-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -174,12 +175,13 @@ body: |
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[BSWAP:%[0-9]+]]:_(<2 x s16>) = G_BSWAP [[BITCAST]]
; GFX8-NEXT: [[BSWAP1:%[0-9]+]]:_(<2 x s16>) = G_BSWAP [[BITCAST1]]
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[BSWAP]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[BSWAP1]](<2 x s16>)
- ; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](s32)
- ; GFX8-NEXT: $vgpr1 = COPY [[LSHR]](s32)
- ; GFX8-NEXT: $vgpr2 = COPY [[BITCAST3]](s32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[BSWAP]](<2 x s16>)
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C3]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[BSWAP1]](<2 x s16>)
+ ; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](i32)
+ ; GFX8-NEXT: $vgpr1 = COPY [[LSHR]](i32)
+ ; GFX8-NEXT: $vgpr2 = COPY [[BITCAST3]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s32) = COPY $vgpr2
@@ -209,11 +211,11 @@ body: |
; GFX7-NEXT: {{ $}}
; GFX7-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX7-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX7-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
@@ -247,12 +249,13 @@ body: |
; GFX7-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[TRUNC7]], [[TRUNC6]]
; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX7-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX7-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
; GFX7-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; GFX7-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX7-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C3]](s32)
; GFX7-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctls.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctls.mir
index 45faa260fdf84..cd166299d4b35 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctls.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctls.mir
@@ -88,22 +88,23 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; CHECK-NEXT: [[INT:%[0-9]+]]:_(i32) = G_INTRINSIC intrinsic(@llvm.amdgcn.sffbh), [[SEXT_INREG]](s32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
; CHECK-NEXT: [[UMIN:%[0-9]+]]:_(i32) = G_UMIN [[INT]], [[C1]]
; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UMIN]], [[C2]]
- ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C3]]
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[SUB1]](s32)
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; CHECK-NEXT: [[INT1:%[0-9]+]]:_(i32) = G_INTRINSIC intrinsic(@llvm.amdgcn.sffbh), [[SEXT_INREG1]](s32)
; CHECK-NEXT: [[UMIN1:%[0-9]+]]:_(i32) = G_UMIN [[INT1]], [[C1]]
; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UMIN1]], [[C2]]
- ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[C]]
+ ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SUB2]], [[C3]]
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[SUB3]](s32)
; CHECK-NEXT: $vgpr0 = COPY [[COPY1]](s32)
; CHECK-NEXT: $vgpr1 = COPY [[COPY2]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-poison.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-poison.mir
index 7d0a59ded53e8..96ddad5f6a781 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-poison.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-poison.mir
@@ -144,18 +144,18 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[BITCAST]], [[C1]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[AMDGPU_FFBH_U32_:%[0-9]+]]:_(i32) = G_AMDGPU_FFBH_U32 [[SHL]](i32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; CHECK-NEXT: [[AMDGPU_FFBH_U32_1:%[0-9]+]]:_(i32) = G_AMDGPU_FFBH_U32 [[SHL1]](i32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_FFBH_U32_]], [[C2]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_FFBH_U32_1]], [[C2]]
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_FFBH_U32_]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_FFBH_U32_1]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz.mir
index dc3e7dd757121..6e2c0dcf65cca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz.mir
@@ -162,23 +162,24 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AMDGPU_FFBH_U32_:%[0-9]+]]:_(s32) = G_AMDGPU_FFBH_U32 [[AND]](s32)
; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
; CHECK-NEXT: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[AMDGPU_FFBH_U32_]], [[C2]]
- ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = nuw G_SUB [[UMIN]], [[C]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = nuw G_SUB [[UMIN]], [[C3]]
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[SUB]](s32)
- ; CHECK-NEXT: [[AMDGPU_FFBH_U32_1:%[0-9]+]]:_(s32) = G_AMDGPU_FFBH_U32 [[LSHR]](s32)
+ ; CHECK-NEXT: [[AMDGPU_FFBH_U32_1:%[0-9]+]]:_(s32) = G_AMDGPU_FFBH_U32 [[LSHR]](i32)
; CHECK-NEXT: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[AMDGPU_FFBH_U32_1]], [[C2]]
- ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = nuw G_SUB [[UMIN1]], [[C]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = nuw G_SUB [[UMIN1]], [[C3]]
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[SUB1]](s32)
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C3]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctpop.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctpop.mir
index 852483a19fb4d..0c08771d247ef 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctpop.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctpop.mir
@@ -185,16 +185,17 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[CTPOP:%[0-9]+]]:_(s32) = G_CTPOP [[AND]](s32)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[CTPOP]](s32)
- ; CHECK-NEXT: [[CTPOP1:%[0-9]+]]:_(s32) = G_CTPOP [[LSHR]](s32)
+ ; CHECK-NEXT: [[CTPOP1:%[0-9]+]]:_(s32) = G_CTPOP [[LSHR]](i32)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[CTPOP1]](s32)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY1]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz-zero-poison.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz-zero-poison.mir
index 3142b11ad7ccf..c5646ba212f58 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz-zero-poison.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz-zero-poison.mir
@@ -141,14 +141,15 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[CTTZ_ZERO_POISON:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_POISON [[BITCAST]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[CTTZ_ZERO_POISON:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_POISON [[BITCAST]](i32)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[CTTZ_ZERO_POISON]](s32)
- ; CHECK-NEXT: [[CTTZ_ZERO_POISON1:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_POISON [[LSHR]](s32)
+ ; CHECK-NEXT: [[CTTZ_ZERO_POISON1:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_POISON [[LSHR]](i32)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[CTTZ_ZERO_POISON1]](s32)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C1]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY1]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz.mir
index 771bbac611501..fe4acfc330eff 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz.mir
@@ -157,9 +157,9 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65536
; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[BITCAST]], [[C1]]
; CHECK-NEXT: [[CTTZ_ZERO_POISON:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_POISON [[OR]](s32)
@@ -167,7 +167,8 @@ body: |
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[C1]]
; CHECK-NEXT: [[CTTZ_ZERO_POISON1:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_POISON [[OR1]](s32)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[CTTZ_ZERO_POISON1]](s32)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[COPY1]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir
index dbcce8e90385b..f31e88834f807 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir
@@ -988,8 +988,8 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](i32)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(s16) = G_EXTRACT %0, 0
%2:_(s32) = G_ANYEXT %1
@@ -1046,10 +1046,10 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(s16) = G_EXTRACT %0, 16
%2:_(s32) = G_ANYEXT %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir
index d7a4b5a5b2186..dc04a5a618454 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir
@@ -275,15 +275,16 @@ body: |
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY]](i32)
; SI-NEXT: [[FABS:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BITCAST]]
; SI-NEXT: [[FABS1:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BITCAST1]]
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[FABS]](<2 x s16>)
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[FABS1]](<2 x s16>)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C2]]
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FABS]](<2 x s16>)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C2]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FABS1]](<2 x s16>)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C3]]
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL1]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C2]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C3]]
; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL2]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
@@ -303,15 +304,16 @@ body: |
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY]](i32)
; VI-NEXT: [[FABS:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BITCAST]]
; VI-NEXT: [[FABS1:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BITCAST1]]
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[FABS]](<2 x s16>)
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[FABS1]](<2 x s16>)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C2]]
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FABS]](<2 x s16>)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FABS1]](<2 x s16>)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C3]]
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL1]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C2]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C3]]
; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL2]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
@@ -327,13 +329,13 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX9-NEXT: [[FABS:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BUILD_VECTOR]]
; GFX9-NEXT: [[FABS1:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BUILD_VECTOR1]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[FABS]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[FABS1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FABS]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FABS1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC]](s16)
; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
index 5d2d20d1ce4a1..43b8aa9e371ee 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
@@ -328,15 +328,15 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
; SI-NEXT: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT]], [[FPEXT1]]
@@ -347,7 +347,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -357,20 +358,21 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[TRUNC1]], [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -399,20 +401,20 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; SI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT]], [[FPEXT1]]
@@ -427,15 +429,16 @@ body: |
; SI-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD2]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -446,34 +449,35 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; VI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; VI-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC3]]
; VI-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[TRUNC1]], [[TRUNC4]]
; VI-NEXT: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[TRUNC2]], [[TRUNC5]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -488,13 +492,13 @@ body: |
; GFX9-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
; GFX9-NEXT: [[FADD:%[0-9]+]]:_(<2 x s16>) = G_FADD [[UV]], [[UV3]]
; GFX9-NEXT: [[FADD1:%[0-9]+]]:_(<2 x s16>) = G_FADD [[UV1]], [[UV4]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[FADD]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[FADD1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
@@ -523,24 +527,24 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
; SI-NEXT: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT]], [[FPEXT1]]
@@ -559,12 +563,13 @@ body: |
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -576,36 +581,37 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC4]]
; VI-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[TRUNC2]], [[TRUNC6]]
; VI-NEXT: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[TRUNC3]], [[TRUNC7]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FADD3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
index 849bebb93d80f..2c56350abbb90 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
@@ -237,11 +237,11 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT]]
; SI-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE]](s32)
@@ -250,7 +250,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -259,16 +260,17 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC1]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FCANONICALIZE]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FCANONICALIZE1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -323,11 +325,11 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX9-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[BUILD_VECTOR]]
; GFX9-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[BUILD_VECTOR1]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[FCANONICALIZE]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[FCANONICALIZE1]](<2 x s16>)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE1]](<2 x s16>)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[BITCAST1]](i32)
; GFX9-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x s32>)
%0:_(<3 x s16>) = G_IMPLICIT_DEF
%1:_(<3 x s16>) = G_FCANONICALIZE %0
@@ -346,15 +348,15 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT]]
; SI-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE]](s32)
@@ -369,12 +371,13 @@ body: |
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE3]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -385,27 +388,28 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC1]]
; VI-NEXT: [[FCANONICALIZE2:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC2]]
; VI-NEXT: [[FCANONICALIZE3:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FCANONICALIZE]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FCANONICALIZE1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FCANONICALIZE2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FCANONICALIZE3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fceil.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fceil.mir
index d819d380b747b..67f904f8ccba4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fceil.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fceil.mir
@@ -172,11 +172,11 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FCEIL:%[0-9]+]]:_(s32) = G_FCEIL [[FPEXT]]
; SI-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCEIL]](s32)
@@ -185,7 +185,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCEIL1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -194,11 +195,11 @@ body: |
; CI: liveins: $vgpr0
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; CI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; CI-NEXT: [[FCEIL:%[0-9]+]]:_(s32) = G_FCEIL [[FPEXT]]
; CI-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCEIL]](s32)
@@ -207,7 +208,8 @@ body: |
; CI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCEIL1]](s32)
; CI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -216,16 +218,17 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[FCEIL:%[0-9]+]]:_(s16) = G_FCEIL [[TRUNC]]
; VI-NEXT: [[FCEIL1:%[0-9]+]]:_(s16) = G_FCEIL [[TRUNC1]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FCEIL]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FCEIL1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -234,11 +237,11 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[FCEIL:%[0-9]+]]:_(s16) = G_FCEIL [[TRUNC]]
; GFX9-NEXT: [[FCEIL1:%[0-9]+]]:_(s16) = G_FCEIL [[TRUNC1]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FCEIL]](s16), [[FCEIL1]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcmp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcmp.mir
index cef4cd7bf238f..2622440c90a83 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcmp.mir
@@ -373,15 +373,15 @@ body: |
; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
; GFX7-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
- ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX7-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX7-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; GFX7-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
; GFX7-NEXT: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[FPEXT]](s32), [[FPEXT1]]
@@ -402,15 +402,15 @@ body: |
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[TRUNC]](s16), [[TRUNC2]]
; GFX8-NEXT: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[TRUNC1]](s16), [[TRUNC3]]
; GFX8-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
@@ -427,15 +427,15 @@ body: |
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
; GFX9-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[TRUNC]](s16), [[TRUNC2]]
; GFX9-NEXT: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[TRUNC1]](s16), [[TRUNC3]]
; GFX9-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcos.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcos.mir
index bba001ef0076e..4f7a276d70866 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcos.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcos.mir
@@ -308,11 +308,11 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float f0x3E22F983
; SI-NEXT: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[C1]]
@@ -326,7 +326,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT3]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -335,11 +336,11 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 1.591800e-01
; VI-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[C1]]
; VI-NEXT: [[INT:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL]](s16)
@@ -349,7 +350,8 @@ body: |
; VI-NEXT: [[INT3:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.cos), [[INT2]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INT1]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INT3]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -358,11 +360,11 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 1.591800e-01
; GFX9-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[C1]]
; GFX9-NEXT: [[INT:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.cos), [[FMUL]](s16)
@@ -453,15 +455,15 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float f0x3E22F983
; SI-NEXT: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[C1]]
@@ -485,12 +487,13 @@ body: |
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT7]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -501,15 +504,15 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 1.591800e-01
; VI-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[C1]]
; VI-NEXT: [[INT:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL]](s16)
@@ -525,12 +528,13 @@ body: |
; VI-NEXT: [[INT7:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.cos), [[INT6]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INT1]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INT3]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[INT5]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[INT7]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -541,15 +545,15 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 1.591800e-01
; GFX9-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[C1]]
; GFX9-NEXT: [[INT:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.cos), [[FMUL]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir
index 687c3985234e3..bf4292770e191 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir
@@ -1743,15 +1743,15 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
; SI-NEXT: [[C1:%[0-9]+]]:_(f32) = G_FCONSTANT float 1.000000e+00
@@ -1785,7 +1785,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT13]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -1795,15 +1796,15 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC]](s16)
; VI-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC2]](s16)
; VI-NEXT: [[FNEG:%[0-9]+]]:_(f32) = G_FNEG [[FPEXT1]]
@@ -1843,7 +1844,8 @@ body: |
; VI-NEXT: [[INT3:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fixup), [[FPTRUNC1]](f16), [[TRUNC3]](s16), [[TRUNC1]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INT1]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INT3]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
@@ -1853,15 +1855,15 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC]](s16)
; GFX9-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC2]](s16)
; GFX9-NEXT: [[FNEG:%[0-9]+]]:_(f32) = G_FNEG [[FPEXT1]]
@@ -1907,15 +1909,15 @@ body: |
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX10-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC]](s16)
; GFX10-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC2]](s16)
; GFX10-NEXT: [[FNEG:%[0-9]+]]:_(f32) = G_FNEG [[FPEXT1]]
@@ -2333,24 +2335,24 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
; SI-NEXT: [[C1:%[0-9]+]]:_(f32) = G_FCONSTANT float 1.000000e+00
@@ -2414,12 +2416,13 @@ body: |
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT27]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -2431,24 +2434,24 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC]](s16)
; VI-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC4]](s16)
; VI-NEXT: [[FNEG:%[0-9]+]]:_(f32) = G_FNEG [[FPEXT1]]
@@ -2524,12 +2527,13 @@ body: |
; VI-NEXT: [[INT7:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fixup), [[FPTRUNC3]](f16), [[TRUNC7]](s16), [[TRUNC3]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INT1]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INT3]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST12:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[INT5]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[INT7]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST13:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST12]](<2 x s16>), [[BITCAST13]](<2 x s16>)
@@ -2541,24 +2545,24 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC]](s16)
; GFX9-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC4]](s16)
; GFX9-NEXT: [[FNEG:%[0-9]+]]:_(f32) = G_FNEG [[FPEXT1]]
@@ -2643,24 +2647,24 @@ body: |
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX10-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX10-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX10-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC]](s16)
; GFX10-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC4]](s16)
; GFX10-NEXT: [[FNEG:%[0-9]+]]:_(f32) = G_FNEG [[FPEXT1]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp2.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp2.mir
index 08055575d1d73..305d1a8a22fba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp2.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp2.mir
@@ -305,11 +305,11 @@ body: |
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[INT:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.exp2), [[FPEXT]](f32)
; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT]](f32)
@@ -318,7 +318,8 @@ body: |
; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT1]](f32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -327,16 +328,17 @@ body: |
; GFX8: liveins: $vgpr0
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX8-NEXT: [[FEXP2_:%[0-9]+]]:_(s16) = G_FEXP2 [[TRUNC]]
; GFX8-NEXT: [[FEXP2_1:%[0-9]+]]:_(s16) = G_FEXP2 [[TRUNC1]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FEXP2_]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FEXP2_1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -345,11 +347,11 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[FEXP2_:%[0-9]+]]:_(s16) = G_FEXP2 [[TRUNC]]
; GFX9-NEXT: [[FEXP2_1:%[0-9]+]]:_(s16) = G_FEXP2 [[TRUNC1]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FEXP2_]](s16), [[FEXP2_1]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ffloor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ffloor.mir
index bdb762fc30d1a..b903ab5d9a88e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ffloor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ffloor.mir
@@ -333,11 +333,11 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FFLOOR:%[0-9]+]]:_(s32) = G_FFLOOR [[FPEXT]]
; SI-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FFLOOR]](s32)
@@ -346,7 +346,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FFLOOR1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -355,16 +356,17 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[FFLOOR:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC]]
; VI-NEXT: [[FFLOOR1:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC1]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FFLOOR]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FFLOOR1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -373,11 +375,11 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[FFLOOR:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC]]
; GFX9-NEXT: [[FFLOOR1:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC1]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FFLOOR]](s16), [[FFLOOR1]](s16)
@@ -447,15 +449,15 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FFLOOR:%[0-9]+]]:_(s32) = G_FFLOOR [[FPEXT]]
; SI-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FFLOOR]](s32)
@@ -470,12 +472,13 @@ body: |
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FFLOOR3]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -486,27 +489,28 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FFLOOR:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC]]
; VI-NEXT: [[FFLOOR1:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC1]]
; VI-NEXT: [[FFLOOR2:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC2]]
; VI-NEXT: [[FFLOOR3:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FFLOOR]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FFLOOR1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FFLOOR2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FFLOOR3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -517,15 +521,15 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[FFLOOR:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC]]
; GFX9-NEXT: [[FFLOOR1:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC1]]
; GFX9-NEXT: [[FFLOOR2:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC2]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog.mir
index 84dabb45f80d9..ae30ebc421e74 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog.mir
@@ -235,11 +235,11 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; CHECK-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC]](s16)
; CHECK-NEXT: [[INT:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.log), [[FPEXT]](f32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(f32) = G_FCONSTANT float f0x3F317218
@@ -251,7 +251,8 @@ body: |
; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](f32)
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog10.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog10.mir
index 84edb0a103f48..9609f2b804dc9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog10.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog10.mir
@@ -235,11 +235,11 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; CHECK-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC]](s16)
; CHECK-NEXT: [[INT:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.log), [[FPEXT]](f32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(f32) = G_FCONSTANT float 3.010300e-01
@@ -251,7 +251,8 @@ body: |
; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](f32)
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
index 803ef369a1982..5aaec886bdd2b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
@@ -379,19 +379,19 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; SI-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
; SI-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
@@ -404,7 +404,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -415,24 +416,25 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; VI-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; VI-NEXT: [[FMA:%[0-9]+]]:_(s16) = G_FMA [[TRUNC]], [[TRUNC2]], [[TRUNC4]]
; VI-NEXT: [[FMA1:%[0-9]+]]:_(s16) = G_FMA [[TRUNC1]], [[TRUNC3]], [[TRUNC5]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMA]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMA1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -465,27 +467,27 @@ body: |
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; SI-NEXT: [[COPY2:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr6_vgpr7_vgpr8
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; SI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; SI-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<6 x s16>)
- ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV6]](<2 x s16>)
- ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; SI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV6]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC6]](s16)
@@ -503,15 +505,16 @@ body: |
; SI-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA2]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>), [[BITCAST8]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -523,41 +526,42 @@ body: |
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; VI-NEXT: [[COPY2:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr6_vgpr7_vgpr8
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; VI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; VI-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<6 x s16>)
- ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV6]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; VI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV6]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
; VI-NEXT: [[FMA:%[0-9]+]]:_(s16) = G_FMA [[TRUNC]], [[TRUNC3]], [[TRUNC6]]
; VI-NEXT: [[FMA1:%[0-9]+]]:_(s16) = G_FMA [[TRUNC1]], [[TRUNC4]], [[TRUNC7]]
; VI-NEXT: [[FMA2:%[0-9]+]]:_(s16) = G_FMA [[TRUNC2]], [[TRUNC5]], [[TRUNC8]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMA]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMA1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMA2]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>), [[BITCAST8]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -574,13 +578,13 @@ body: |
; GFX9-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<6 x s16>)
; GFX9-NEXT: [[FMA:%[0-9]+]]:_(<2 x s16>) = G_FMA [[UV]], [[UV3]], [[UV6]]
; GFX9-NEXT: [[FMA1:%[0-9]+]]:_(<2 x s16>) = G_FMA [[UV1]], [[UV4]], [[UV7]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[FMA]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[FMA1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
@@ -612,33 +616,33 @@ body: |
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; SI-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; SI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; SI-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; SI-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; SI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; SI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
; SI-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC8]](s16)
@@ -661,12 +665,13 @@ body: |
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA3]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -679,45 +684,46 @@ body: |
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; VI-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; VI-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; VI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; VI-NEXT: [[FMA:%[0-9]+]]:_(s16) = G_FMA [[TRUNC]], [[TRUNC4]], [[TRUNC8]]
; VI-NEXT: [[FMA1:%[0-9]+]]:_(s16) = G_FMA [[TRUNC1]], [[TRUNC5]], [[TRUNC9]]
; VI-NEXT: [[FMA2:%[0-9]+]]:_(s16) = G_FMA [[TRUNC2]], [[TRUNC6]], [[TRUNC10]]
; VI-NEXT: [[FMA3:%[0-9]+]]:_(s16) = G_FMA [[TRUNC3]], [[TRUNC7]], [[TRUNC11]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMA]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMA1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMA2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMA3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir
index d3232eed8949f..323adfcf35311 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir
@@ -89,19 +89,19 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX6-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
; GFX6-NEXT: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
@@ -120,7 +120,8 @@ body: |
; GFX6-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -131,24 +132,25 @@ body: |
; GFX7-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX7-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX7-NEXT: [[FMAD:%[0-9]+]]:_(s16) = G_FMAD [[TRUNC]], [[TRUNC2]], [[TRUNC4]]
; GFX7-NEXT: [[FMAD1:%[0-9]+]]:_(s16) = G_FMAD [[TRUNC1]], [[TRUNC3]], [[TRUNC5]]
; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAD]](s16)
; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAD1]](s16)
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX7-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -159,19 +161,19 @@ body: |
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX10-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC2]]
; GFX10-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC4]]
; GFX10-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC3]]
@@ -203,33 +205,33 @@ body: |
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX6-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX6-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX6-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
; GFX6-NEXT: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
@@ -264,12 +266,13 @@ body: |
; GFX6-NEXT: [[FPTRUNC7:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC5]](s16)
; GFX6-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC7]](s16)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -282,45 +285,46 @@ body: |
; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; GFX7-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX7-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX7-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX7-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX7-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX7-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; GFX7-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; GFX7-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; GFX7-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; GFX7-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; GFX7-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; GFX7-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; GFX7-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; GFX7-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; GFX7-NEXT: [[FMAD:%[0-9]+]]:_(s16) = G_FMAD [[TRUNC]], [[TRUNC4]], [[TRUNC8]]
; GFX7-NEXT: [[FMAD1:%[0-9]+]]:_(s16) = G_FMAD [[TRUNC1]], [[TRUNC5]], [[TRUNC9]]
; GFX7-NEXT: [[FMAD2:%[0-9]+]]:_(s16) = G_FMAD [[TRUNC2]], [[TRUNC6]], [[TRUNC10]]
; GFX7-NEXT: [[FMAD3:%[0-9]+]]:_(s16) = G_FMAD [[TRUNC3]], [[TRUNC7]], [[TRUNC11]]
; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAD]](s16)
; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAD1]](s16)
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMAD2]](s16)
; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMAD3]](s16)
- ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX7-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX7-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -333,33 +337,33 @@ body: |
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; GFX10-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX10-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX10-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; GFX10-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; GFX10-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; GFX10-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; GFX10-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; GFX10-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC4]]
; GFX10-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC8]]
; GFX10-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC5]]
@@ -535,19 +539,19 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX6-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
; GFX6-NEXT: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
@@ -566,7 +570,8 @@ body: |
; GFX6-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -577,26 +582,27 @@ body: |
; GFX7-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX7-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX7-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC2]]
; GFX7-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC4]]
; GFX7-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC3]]
; GFX7-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[FMUL1]], [[TRUNC5]]
; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX7-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -607,19 +613,19 @@ body: |
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX10-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC2]]
; GFX10-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC4]]
; GFX10-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC3]]
@@ -650,19 +656,19 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX6-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC2]](s16)
; GFX6-NEXT: [[FMUL:%[0-9]+]]:_(s32) = nnan G_FMUL [[FPEXT]], [[FPEXT1]]
@@ -681,7 +687,8 @@ body: |
; GFX6-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FADD1]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -692,26 +699,27 @@ body: |
; GFX7-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX7-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX7-NEXT: [[FMUL:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC]], [[TRUNC2]]
; GFX7-NEXT: [[FADD:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL]], [[TRUNC4]]
; GFX7-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC1]], [[TRUNC3]]
; GFX7-NEXT: [[FADD1:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL1]], [[TRUNC5]]
; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX7-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -722,19 +730,19 @@ body: |
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX10-NEXT: [[FMUL:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC]], [[TRUNC2]]
; GFX10-NEXT: [[FADD:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL]], [[TRUNC4]]
; GFX10-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC1]], [[TRUNC3]]
@@ -766,33 +774,33 @@ body: |
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX6-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX6-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX6-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
; GFX6-NEXT: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
@@ -827,12 +835,13 @@ body: |
; GFX6-NEXT: [[FPTRUNC7:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC5]](s16)
; GFX6-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC7]](s16)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -845,33 +854,33 @@ body: |
; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; GFX7-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX7-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX7-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX7-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX7-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX7-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; GFX7-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; GFX7-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; GFX7-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; GFX7-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; GFX7-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; GFX7-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; GFX7-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; GFX7-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; GFX7-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC4]]
; GFX7-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC8]]
; GFX7-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC5]]
@@ -882,12 +891,13 @@ body: |
; GFX7-NEXT: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[FMUL3]], [[TRUNC11]]
; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FADD3]](s16)
- ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX7-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX7-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -900,33 +910,33 @@ body: |
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; GFX10-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX10-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX10-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; GFX10-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; GFX10-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; GFX10-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; GFX10-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; GFX10-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC4]]
; GFX10-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC8]]
; GFX10-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC5]]
@@ -965,33 +975,33 @@ body: |
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX6-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX6-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX6-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC4]](s16)
; GFX6-NEXT: [[FMUL:%[0-9]+]]:_(s32) = nnan G_FMUL [[FPEXT]], [[FPEXT1]]
@@ -1026,12 +1036,13 @@ body: |
; GFX6-NEXT: [[FPTRUNC7:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FADD3]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC5]](s16)
; GFX6-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC7]](s16)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -1044,33 +1055,33 @@ body: |
; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; GFX7-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX7-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX7-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX7-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX7-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX7-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; GFX7-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; GFX7-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; GFX7-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; GFX7-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; GFX7-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; GFX7-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; GFX7-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; GFX7-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; GFX7-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; GFX7-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; GFX7-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; GFX7-NEXT: [[FMUL:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC]], [[TRUNC4]]
; GFX7-NEXT: [[FADD:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL]], [[TRUNC8]]
; GFX7-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC1]], [[TRUNC5]]
@@ -1081,12 +1092,13 @@ body: |
; GFX7-NEXT: [[FADD3:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL3]], [[TRUNC11]]
; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FADD3]](s16)
- ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX7-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX7-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -1099,33 +1111,33 @@ body: |
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; GFX10-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX10-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX10-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; GFX10-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; GFX10-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; GFX10-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; GFX10-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; GFX10-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; GFX10-NEXT: [[FMUL:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC]], [[TRUNC4]]
; GFX10-NEXT: [[FADD:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL]], [[TRUNC8]]
; GFX10-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC1]], [[TRUNC5]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaximum.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaximum.mir
index 0f3b2753710b9..d0888953dd7c2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaximum.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaximum.mir
@@ -122,22 +122,22 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(<2 x s16>) = G_FMAXNUM_IEEE [[COPY]], [[COPY1]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(ord), [[TRUNC]](s16), [[TRUNC2]]
; GFX9-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(ord), [[TRUNC1]](s16), [[TRUNC3]]
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half +qnan
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[FMAXNUM_IEEE]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMAXNUM_IEEE]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(s16) = G_SELECT [[FCMP]](i1), [[TRUNC4]], [[C1]]
; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(s16) = G_SELECT [[FCMP1]](i1), [[TRUNC5]], [[C1]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[SELECT]](s16), [[SELECT1]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaxnum.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaxnum.mir
index 0914efa37ae94..30bf9b2cfa03e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaxnum.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaxnum.mir
@@ -400,15 +400,15 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
; SI-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT]], [[FPEXT1]]
@@ -419,7 +419,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -429,15 +430,15 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC2]]
; VI-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]]
@@ -446,7 +447,8 @@ body: |
; VI-NEXT: [[FMAXNUM_IEEE1:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE2]], [[FCANONICALIZE3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -477,20 +479,20 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; SI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT]], [[FPEXT1]]
@@ -505,15 +507,16 @@ body: |
; SI-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE2]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -524,20 +527,20 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; VI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC3]]
; VI-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]]
@@ -549,15 +552,16 @@ body: |
; VI-NEXT: [[FMAXNUM_IEEE2:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE4]], [[FCANONICALIZE5]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE2]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -576,13 +580,13 @@ body: |
; GFX9-NEXT: [[FCANONICALIZE2:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[UV1]]
; GFX9-NEXT: [[FCANONICALIZE3:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[UV4]]
; GFX9-NEXT: [[FMAXNUM_IEEE1:%[0-9]+]]:_(<2 x s16>) = G_FMAXNUM_IEEE [[FCANONICALIZE2]], [[FCANONICALIZE3]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[FMAXNUM_IEEE]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[FMAXNUM_IEEE1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FMAXNUM_IEEE]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FMAXNUM_IEEE1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
@@ -611,24 +615,24 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
; SI-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT]], [[FPEXT1]]
@@ -647,12 +651,13 @@ body: |
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE3]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -664,24 +669,24 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC4]]
; VI-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]]
@@ -696,12 +701,13 @@ body: |
; VI-NEXT: [[FMAXNUM_IEEE3:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE6]], [[FCANONICALIZE7]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -997,11 +1003,11 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0.000000e+00
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[C]](s16)
; SI-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT]], [[FPEXT1]]
@@ -1012,7 +1018,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -1022,11 +1029,11 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0.000000e+00
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[C]]
; VI-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]]
@@ -1035,7 +1042,8 @@ body: |
; VI-NEXT: [[FMAXNUM_IEEE1:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE2]], [[FCANONICALIZE3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminimum.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminimum.mir
index 7db29e22215ca..718360a4742ba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminimum.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminimum.mir
@@ -122,22 +122,22 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(<2 x s16>) = G_FMINNUM_IEEE [[COPY]], [[COPY1]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(ord), [[TRUNC]](s16), [[TRUNC2]]
; GFX9-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(ord), [[TRUNC1]](s16), [[TRUNC3]]
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half +qnan
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[FMINNUM_IEEE]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMINNUM_IEEE]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(s16) = G_SELECT [[FCMP]](i1), [[TRUNC4]], [[C1]]
; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(s16) = G_SELECT [[FCMP1]](i1), [[TRUNC5]], [[C1]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[SELECT]](s16), [[SELECT1]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminnum.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminnum.mir
index 5ee21f3e0aa4d..b94d95d0bb2af 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminnum.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminnum.mir
@@ -400,15 +400,15 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
; SI-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FPEXT]], [[FPEXT1]]
@@ -419,7 +419,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMINNUM_IEEE1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -429,15 +430,15 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC2]]
; VI-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(s16) = G_FMINNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]]
@@ -446,7 +447,8 @@ body: |
; VI-NEXT: [[FMINNUM_IEEE1:%[0-9]+]]:_(s16) = G_FMINNUM_IEEE [[FCANONICALIZE2]], [[FCANONICALIZE3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -477,20 +479,20 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; SI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FPEXT]], [[FPEXT1]]
@@ -505,15 +507,16 @@ body: |
; SI-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMINNUM_IEEE2]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -524,20 +527,20 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; VI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC3]]
; VI-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(s16) = G_FMINNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]]
@@ -549,15 +552,16 @@ body: |
; VI-NEXT: [[FMINNUM_IEEE2:%[0-9]+]]:_(s16) = G_FMINNUM_IEEE [[FCANONICALIZE4]], [[FCANONICALIZE5]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE2]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -576,13 +580,13 @@ body: |
; GFX9-NEXT: [[FCANONICALIZE2:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[UV1]]
; GFX9-NEXT: [[FCANONICALIZE3:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[UV4]]
; GFX9-NEXT: [[FMINNUM_IEEE1:%[0-9]+]]:_(<2 x s16>) = G_FMINNUM_IEEE [[FCANONICALIZE2]], [[FCANONICALIZE3]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[FMINNUM_IEEE]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[FMINNUM_IEEE1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FMINNUM_IEEE]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FMINNUM_IEEE1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
@@ -611,24 +615,24 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
; SI-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FPEXT]], [[FPEXT1]]
@@ -647,12 +651,13 @@ body: |
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMINNUM_IEEE3]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -664,24 +669,24 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC4]]
; VI-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(s16) = G_FMINNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]]
@@ -696,12 +701,13 @@ body: |
; VI-NEXT: [[FMINNUM_IEEE3:%[0-9]+]]:_(s16) = G_FMINNUM_IEEE [[FCANONICALIZE6]], [[FCANONICALIZE7]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -997,11 +1003,11 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0.000000e+00
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[C]](s16)
; SI-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FPEXT]], [[FPEXT1]]
@@ -1012,7 +1018,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMINNUM_IEEE1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -1022,11 +1029,11 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0.000000e+00
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[C]]
; VI-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(s16) = G_FMINNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]]
@@ -1035,7 +1042,8 @@ body: |
; VI-NEXT: [[FMINNUM_IEEE1:%[0-9]+]]:_(s16) = G_FMINNUM_IEEE [[FCANONICALIZE2]], [[FCANONICALIZE3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
index 6a68441cc3e88..41606c838c59f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
@@ -327,15 +327,15 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
; SI-NEXT: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
@@ -346,7 +346,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -356,20 +357,21 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -399,20 +401,20 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; SI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
@@ -427,15 +429,16 @@ body: |
; SI-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL2]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -446,34 +449,35 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; VI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; VI-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC3]]
; VI-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC4]]
; VI-NEXT: [[FMUL2:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC2]], [[TRUNC5]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL2]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -488,13 +492,13 @@ body: |
; GFX9PLUS-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
; GFX9PLUS-NEXT: [[FMUL:%[0-9]+]]:_(<2 x s16>) = G_FMUL [[UV]], [[UV3]]
; GFX9PLUS-NEXT: [[FMUL1:%[0-9]+]]:_(<2 x s16>) = G_FMUL [[UV1]], [[UV4]]
- ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[FMUL]](<2 x s16>)
- ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9PLUS-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[FMUL1]](<2 x s16>)
- ; GFX9PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](<2 x s16>)
+ ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9PLUS-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](<2 x s16>)
+ ; GFX9PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9PLUS-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
; GFX9PLUS-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
@@ -523,24 +527,24 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
; SI-NEXT: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
@@ -559,12 +563,13 @@ body: |
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL3]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -576,36 +581,37 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC4]]
; VI-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[FMUL2:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC2]], [[TRUNC6]]
; VI-NEXT: [[FMUL3:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC3]], [[TRUNC7]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir
index 67bcc919bf186..a295c82345320 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir
@@ -273,13 +273,14 @@ body: |
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY]](i32)
; SI-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BITCAST]]
; SI-NEXT: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BITCAST1]]
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG]](<2 x s16>)
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG1]](<2 x s16>)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C2]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C2]]
- ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](s32), [[AND1]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG]](<2 x s16>)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C2]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG1]](<2 x s16>)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C3]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C3]]
+ ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](i32), [[AND1]](s32)
; SI-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s32>)
;
; VI-LABEL: name: test_fneg_v3s16
@@ -292,13 +293,14 @@ body: |
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY]](i32)
; VI-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BITCAST]]
; VI-NEXT: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BITCAST1]]
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG]](<2 x s16>)
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG1]](<2 x s16>)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C2]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C2]]
- ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](s32), [[AND1]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG]](<2 x s16>)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG1]](<2 x s16>)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C3]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C3]]
+ ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](i32), [[AND1]](s32)
; VI-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s32>)
;
; GFX9-LABEL: name: test_fneg_v3s16
@@ -307,14 +309,14 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX9-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BUILD_VECTOR]]
; GFX9-NEXT: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BUILD_VECTOR1]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG1]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG1]](<2 x s16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX9-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](s32), [[AND1]](s32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](i32), [[AND1]](s32)
; GFX9-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x s32>)
%0:_(<3 x s16>) = G_IMPLICIT_DEF
%1:_(<3 x s16>) = G_FNEG %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir
index cc3cf5d359714..a9a3ac514db3d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir
@@ -30,11 +30,11 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; CHECK-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC]](s16)
; CHECK-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC1]](s16)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPEXT]](s32), [[FPEXT1]](s32)
@@ -54,11 +54,11 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; CHECK-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC]](s16)
; CHECK-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC1]](s16)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPEXT]](s32), [[FPEXT1]](s32)
@@ -78,13 +78,13 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; CHECK-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; CHECK-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
; CHECK-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
@@ -228,11 +228,11 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; CHECK-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC]](s16)
; CHECK-NEXT: [[FPEXT1:%[0-9]+]]:_(s64) = G_FPEXT [[FPEXT]](s32)
; CHECK-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC1]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir
index 9f21d91167187..fa944cbed55cc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir
@@ -445,15 +445,15 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
; GFX6-NEXT: [[INT:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.log), [[FPEXT]](s32)
@@ -483,7 +483,8 @@ body: |
; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -493,15 +494,15 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[FLOG2_:%[0-9]+]]:_(f16) = G_FLOG2 [[TRUNC]]
; GFX9-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[FLOG2_]](f16)
; GFX9-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC2]](s16)
@@ -533,15 +534,15 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = nnan nsz G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = nnan nsz G_FPEXT [[TRUNC2]](s16)
; GFX6-NEXT: [[INT:%[0-9]+]]:_(f32) = nnan nsz G_INTRINSIC intrinsic(@llvm.amdgcn.log), [[FPEXT]](s32)
@@ -571,7 +572,8 @@ body: |
; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = nnan nsz G_FPTRUNC [[FMUL1]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -581,15 +583,15 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[FLOG2_:%[0-9]+]]:_(f16) = nnan nsz G_FLOG2 [[TRUNC]]
; GFX9-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = nnan nsz G_FPEXT [[FLOG2_]](f16)
; GFX9-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = nnan nsz G_FPEXT [[TRUNC2]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir
index 8998c566318c8..7b051bb6ad1e9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir
@@ -622,11 +622,11 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[FPEXT]](s32)
; SI-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[FPTOSI]](s32)
@@ -640,11 +640,11 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; VI-NEXT: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[FPEXT]](s32)
; VI-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[FPTOSI]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir
index 7014bda3385e6..30777fc92c96e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir
@@ -564,11 +564,11 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FPEXT]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[FPTOUI]](s32)
@@ -582,11 +582,11 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; VI-NEXT: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FPEXT]](s32)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[FPTOUI]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir
index f8800f8061978..3f12e9381c67b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir
@@ -76,9 +76,10 @@ body: |
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x f16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BITCAST1]](s32), [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x f16>)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BITCAST1]](i32), [[LSHR]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
%0:_(<2 x f32>) = COPY $vgpr0_vgpr1
%1:_(<2 x f16>) = G_FPTRUNC %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
index 7f3682f912a17..f99519fa582bc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
@@ -555,10 +555,11 @@ body: |
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<4 x s16>) = G_FREEZE [[CONCAT_VECTORS]]
; CHECK-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[FREEZE]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST2]](s32), [[LSHR]](s32), [[BITCAST3]](s32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C3]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST2]](i32), [[LSHR]](i32), [[BITCAST3]](i32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
%0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
%1:_(<3 x s16>) = G_TRUNC %0
@@ -609,12 +610,13 @@ body: |
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<6 x s16>) = G_FREEZE [[CONCAT_VECTORS]]
; CHECK-NEXT: [[UV5:%[0-9]+]]:_(<2 x s16>), [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[FREEZE]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV6]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[BITCAST3]](s32), [[LSHR]](s32), [[BITCAST4]](s32), [[LSHR1]](s32), [[BITCAST5]](s32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C3]](i32)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV6]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C3]](i32)
+ ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[BITCAST3]](i32), [[LSHR]](i32), [[BITCAST4]](i32), [[LSHR1]](i32), [[BITCAST5]](i32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 = COPY [[BUILD_VECTOR]](<5 x s32>)
%0:_(<5 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
%1:_(<5 x s16>) = G_TRUNC %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir
index 4bc7e90ffb0dd..e168596239147 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir
@@ -234,15 +234,15 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; SI-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
; SI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
; SI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -273,7 +273,8 @@ body: |
; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC4]], [[TRUNC5]]
; SI-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C]](s32)
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C5]](s32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL2]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -284,19 +285,19 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; VI-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
; VI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C1]]
; VI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -316,7 +317,8 @@ body: |
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL1]], [[LSHR6]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -669,19 +671,19 @@ body: |
; SI-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
; SI-NEXT: [[COPY4:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
; SI-NEXT: [[COPY5:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
- ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY4]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[COPY5]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY3]](<2 x s16>)
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[COPY4]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[COPY5]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
; SI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
; SI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -725,16 +727,17 @@ body: |
; SI-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC7]], [[TRUNC8]]
; SI-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; SI-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C]](s32)
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C5]](s32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT6]], [[SHL3]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; SI-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C]](s32)
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT8]], [[SHL4]]
; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C]](s32)
- ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL5]]
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr1 = COPY [[BITCAST7]](<2 x s16>)
@@ -749,25 +752,25 @@ body: |
; VI-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
; VI-NEXT: [[COPY4:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
; VI-NEXT: [[COPY5:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[COPY5]](<2 x s16>)
- ; VI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[COPY4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[COPY5]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
; VI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C1]]
; VI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -794,16 +797,17 @@ body: |
; VI-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL2]], [[LSHR8]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C]](s32)
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C]](s32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL5]]
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL5]]
; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr1 = COPY [[BITCAST7]](<2 x s16>)
@@ -841,8 +845,8 @@ body: |
; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[COPY3]], [[BUILD_VECTOR5]](<2 x s16>)
; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[LSHR2]], [[AND3]](<2 x s16>)
; GFX9-NEXT: [[OR1:%[0-9]+]]:_(<2 x s16>) = disjoint G_OR [[SHL1]], [[LSHR3]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[OR1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[OR1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX9-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
@@ -884,25 +888,25 @@ body: |
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; SI-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; SI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
; SI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
; SI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -958,12 +962,13 @@ body: |
; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC10]], [[TRUNC11]]
; SI-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; SI-NEXT: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C]](s32)
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C5]](s32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT8]], [[SHL4]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
; SI-NEXT: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
; SI-NEXT: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT11]], [[C]](s32)
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT11]], [[C5]](s32)
; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT10]], [[SHL5]]
; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -976,33 +981,33 @@ body: |
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; VI-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; VI-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; VI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
; VI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C1]]
; VI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -1036,12 +1041,13 @@ body: |
; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL3]], [[LSHR13]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C4]](s32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
index b636a45bdc019..b7dd9f82c1e13 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
@@ -196,15 +196,15 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; SI-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
; SI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
; SI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -235,7 +235,8 @@ body: |
; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC4]], [[TRUNC5]]
; SI-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C]](s32)
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C5]](s32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL4]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -246,19 +247,19 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; VI-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
; VI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C1]]
; VI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -278,7 +279,8 @@ body: |
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL3]], [[LSHR4]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -625,19 +627,19 @@ body: |
; SI-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
; SI-NEXT: [[COPY4:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
; SI-NEXT: [[COPY5:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
- ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY4]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[COPY5]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY3]](<2 x s16>)
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[COPY4]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[COPY5]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
; SI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
; SI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -681,16 +683,17 @@ body: |
; SI-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC7]], [[TRUNC8]]
; SI-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; SI-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C]](s32)
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C5]](s32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT6]], [[SHL6]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; SI-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C]](s32)
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT8]], [[SHL7]]
; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; SI-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C]](s32)
- ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL8]]
+ ; SI-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL8]]
; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr1 = COPY [[BITCAST7]](<2 x s16>)
@@ -705,25 +708,25 @@ body: |
; VI-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
; VI-NEXT: [[COPY4:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
; VI-NEXT: [[COPY5:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[COPY5]](<2 x s16>)
- ; VI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[COPY4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[COPY5]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
; VI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C1]]
; VI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -750,16 +753,17 @@ body: |
; VI-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL5]], [[LSHR5]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL6]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C]](s32)
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL7]]
; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C]](s32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL8]]
+ ; VI-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL8]]
; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr1 = COPY [[BITCAST7]](<2 x s16>)
@@ -797,8 +801,8 @@ body: |
; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(<2 x s16>) = G_SHL [[SHL2]], [[AND3]](<2 x s16>)
; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[COPY3]], [[AND2]](<2 x s16>)
; GFX9-NEXT: [[OR1:%[0-9]+]]:_(<2 x s16>) = disjoint G_OR [[SHL3]], [[LSHR1]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[OR1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[OR1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX9-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
@@ -842,15 +846,15 @@ body: |
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
; SI-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
; SI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
; SI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -881,17 +885,18 @@ body: |
; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC4]], [[TRUNC5]]
; SI-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C]](s32)
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C5]](s32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL4]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST6]](s32)
- ; SI-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
- ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](s32)
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST6]](i32)
+ ; SI-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](i32)
; SI-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C1]]
; SI-NEXT: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC6]], [[C2]]
; SI-NEXT: [[AND6:%[0-9]+]]:_(s16) = G_AND [[XOR2]], [[C1]]
@@ -919,7 +924,7 @@ body: |
; SI-NEXT: [[OR4:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC10]], [[TRUNC11]]
; SI-NEXT: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
; SI-NEXT: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
- ; SI-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT11]], [[C]](s32)
+ ; SI-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT11]], [[C5]](s32)
; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT10]], [[SHL9]]
; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -934,19 +939,19 @@ body: |
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
; VI-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
; VI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C1]]
; VI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -966,21 +971,22 @@ body: |
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL3]], [[LSHR4]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
- ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; VI-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; VI-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](s32)
- ; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; VI-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST6]](s32)
- ; VI-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
- ; VI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](s32)
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
+ ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; VI-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](i32)
+ ; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST6]](i32)
+ ; VI-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](i32)
; VI-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC10]], [[C1]]
; VI-NEXT: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC10]], [[C2]]
; VI-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[XOR2]], [[C1]]
@@ -997,7 +1003,7 @@ body: |
; VI-NEXT: [[OR4:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL8]], [[LSHR9]]
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
- ; VI-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C4]](s32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL9]]
; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST7]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsin.mir
index 5714d228e4d22..8cae9895bd362 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsin.mir
@@ -308,11 +308,11 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float f0x3E22F983
; SI-NEXT: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[C1]]
@@ -326,7 +326,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT3]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -335,11 +336,11 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 1.591800e-01
; VI-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[C1]]
; VI-NEXT: [[INT:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL]](s16)
@@ -349,7 +350,8 @@ body: |
; VI-NEXT: [[INT3:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.sin), [[INT2]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INT1]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INT3]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -358,11 +360,11 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 1.591800e-01
; GFX9-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[C1]]
; GFX9-NEXT: [[INT:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.sin), [[FMUL]](s16)
@@ -453,15 +455,15 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float f0x3E22F983
; SI-NEXT: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[C1]]
@@ -485,12 +487,13 @@ body: |
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT7]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -501,15 +504,15 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 1.591800e-01
; VI-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[C1]]
; VI-NEXT: [[INT:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL]](s16)
@@ -525,12 +528,13 @@ body: |
; VI-NEXT: [[INT7:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.sin), [[INT6]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INT1]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INT3]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[INT5]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[INT7]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -541,15 +545,15 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 1.591800e-01
; GFX9-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[C1]]
; GFX9-NEXT: [[INT:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.sin), [[FMUL]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir
index c2d2bc32d5268..ec345d61d3eab 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir
@@ -399,11 +399,11 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[INT:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.sqrt), [[FPEXT]](f32)
; SI-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT]](f32)
@@ -412,7 +412,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT1]](f32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -421,16 +422,17 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC]]
; VI-NEXT: [[FSQRT1:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC1]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FSQRT]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FSQRT1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -439,11 +441,11 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC]]
; GFX9-NEXT: [[FSQRT1:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC1]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FSQRT]](s16), [[FSQRT1]](s16)
@@ -513,15 +515,15 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[INT:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.sqrt), [[FPEXT]](f32)
; SI-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT]](f32)
@@ -536,12 +538,13 @@ body: |
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT3]](f32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -552,27 +555,28 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC]]
; VI-NEXT: [[FSQRT1:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC1]]
; VI-NEXT: [[FSQRT2:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC2]]
; VI-NEXT: [[FSQRT3:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FSQRT]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FSQRT1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FSQRT2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FSQRT3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -583,15 +587,15 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC]]
; GFX9-NEXT: [[FSQRT1:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC1]]
; GFX9-NEXT: [[FSQRT2:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC2]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir
index 48bf3fd809d08..d302ebbd220fd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir
@@ -376,15 +376,15 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; SI-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[COPY1]]
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
; SI-NEXT: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT]], [[FPEXT1]]
@@ -395,7 +395,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -406,20 +407,21 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; VI-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[COPY1]]
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[TRUNC1]], [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -449,21 +451,21 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; SI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
; SI-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[UV3]]
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG]](<2 x s16>)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
; SI-NEXT: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT]], [[FPEXT1]]
@@ -479,15 +481,16 @@ body: |
; SI-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD2]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -498,35 +501,36 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; VI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
; VI-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[UV3]]
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC4]]
; VI-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[FSUB:%[0-9]+]]:_(s16) = G_FSUB [[TRUNC2]], [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FSUB]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -537,19 +541,19 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX9-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[UV3]]
; GFX9-NEXT: [[FADD:%[0-9]+]]:_(<2 x s16>) = G_FADD [[UV]], [[FNEG]]
; GFX9-NEXT: [[FSUB:%[0-9]+]]:_(s16) = G_FSUB [[TRUNC]], [[TRUNC1]]
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[FADD]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FSUB]](s16), [[DEF]](s16)
@@ -581,15 +585,15 @@ body: |
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
; SI-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[UV2]]
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG]](<2 x s16>)
- ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
; SI-NEXT: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT]], [[FPEXT1]]
@@ -600,18 +604,19 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[UV3]]
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG1]](<2 x s16>)
- ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG1]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; SI-NEXT: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
; SI-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC6]](s16)
; SI-NEXT: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FPEXT4]], [[FPEXT5]]
@@ -622,7 +627,7 @@ body: |
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -636,36 +641,37 @@ body: |
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
; VI-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[UV2]]
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[TRUNC1]], [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[UV3]]
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[FNEG1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[TRUNC4]], [[TRUNC6]]
; VI-NEXT: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[TRUNC5]], [[TRUNC7]]
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FADD3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-icmp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-icmp.mir
index 2d5bf68161300..ac0c09bc905cb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-icmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-icmp.mir
@@ -845,16 +845,16 @@ body: |
; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
; GFX7-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
- ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX7-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX7-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; GFX7-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; GFX7-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[AND]](s32), [[AND1]]
- ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[LSHR]](s32), [[LSHR1]]
+ ; GFX7-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[LSHR]](i32), [[LSHR1]]
; GFX7-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
; GFX7-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY3]](<2 x s32>)
; GFX7-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
@@ -869,15 +869,15 @@ body: |
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[TRUNC]](s16), [[TRUNC2]]
; GFX8-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[TRUNC1]](s16), [[TRUNC3]]
; GFX8-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
@@ -894,15 +894,15 @@ body: |
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
; GFX9-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[TRUNC]](s16), [[TRUNC2]]
; GFX9-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[TRUNC1]](s16), [[TRUNC3]]
; GFX9-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir
index dcfc8b0f2ae2d..e00beb5611f7e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir
@@ -1136,12 +1136,13 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -1190,7 +1191,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
@@ -1216,13 +1217,14 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[UV1]](<2 x s16>)
@@ -1247,7 +1249,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C]]
@@ -1330,19 +1332,20 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL1]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -1395,9 +1398,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
@@ -1431,12 +1434,13 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[UV1]](<2 x s16>)
@@ -1459,7 +1463,7 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
@@ -1487,12 +1491,13 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -1515,7 +1520,7 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
@@ -1543,12 +1548,13 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[UV1]](<2 x s16>)
@@ -1570,19 +1576,20 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL1]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
@@ -1604,12 +1611,13 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -1631,14 +1639,15 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -1661,20 +1670,21 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL1]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
@@ -1723,18 +1733,19 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL1]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir
index 780ffeee11bd0..4d036f43ae267 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir
@@ -584,11 +584,11 @@ body: |
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FPEXT]]
; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[INTRINSIC_TRUNC]](s32)
@@ -636,7 +636,8 @@ body: |
; GFX6-NEXT: [[FPTRUNC5:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC5]](s16)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -645,11 +646,11 @@ body: |
; GFX8: liveins: $vgpr0
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX8-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s16) = G_INTRINSIC_TRUNC [[TRUNC]]
; GFX8-NEXT: [[FSUB:%[0-9]+]]:_(s16) = G_FSUB [[TRUNC]], [[INTRINSIC_TRUNC]]
; GFX8-NEXT: [[FABS:%[0-9]+]]:_(s16) = G_FABS [[FSUB]]
@@ -675,7 +676,8 @@ body: |
; GFX8-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[INTRINSIC_TRUNC1]], [[OR1]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -684,11 +686,11 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s16) = G_INTRINSIC_TRUNC [[TRUNC]]
; GFX9-NEXT: [[FSUB:%[0-9]+]]:_(s16) = G_FSUB [[TRUNC]], [[INTRINSIC_TRUNC]]
; GFX9-NEXT: [[FABS:%[0-9]+]]:_(s16) = G_FABS [[FSUB]]
@@ -729,13 +731,13 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FPEXT]]
; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[INTRINSIC_TRUNC]](s32)
@@ -803,15 +805,16 @@ body: |
; GFX6-NEXT: [[FPTRUNC8:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD5]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC5]](s16)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
; GFX6-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC8]](s16)
- ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C]](s32)
+ ; GFX6-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C7]], [[C6]](s32)
; GFX6-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; GFX6-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL1]]
+ ; GFX6-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C7]], [[SHL1]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -821,13 +824,13 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX8-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s16) = G_INTRINSIC_TRUNC [[TRUNC]]
; GFX8-NEXT: [[FSUB:%[0-9]+]]:_(s16) = G_FSUB [[TRUNC]], [[INTRINSIC_TRUNC]]
; GFX8-NEXT: [[FABS:%[0-9]+]]:_(s16) = G_FABS [[FSUB]]
@@ -862,15 +865,16 @@ body: |
; GFX8-NEXT: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[INTRINSIC_TRUNC2]], [[OR2]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
; GFX8-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
- ; GFX8-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C]](s32)
+ ; GFX8-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C7]], [[C6]](s32)
; GFX8-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; GFX8-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL1]]
+ ; GFX8-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C7]], [[SHL1]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -880,13 +884,13 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s16) = G_INTRINSIC_TRUNC [[TRUNC]]
; GFX9-NEXT: [[FSUB:%[0-9]+]]:_(s16) = G_FSUB [[TRUNC]], [[INTRINSIC_TRUNC]]
; GFX9-NEXT: [[FABS:%[0-9]+]]:_(s16) = G_FABS [[FSUB]]
@@ -944,15 +948,15 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FPEXT]]
; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[INTRINSIC_TRUNC]](s32)
@@ -1040,12 +1044,13 @@ body: |
; GFX6-NEXT: [[FPTRUNC11:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD7]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC5]](s16)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
; GFX6-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC8]](s16)
; GFX6-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC11]](s16)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C6]](s32)
; GFX6-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -1056,15 +1061,15 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s16) = G_INTRINSIC_TRUNC [[TRUNC]]
; GFX8-NEXT: [[FSUB:%[0-9]+]]:_(s16) = G_FSUB [[TRUNC]], [[INTRINSIC_TRUNC]]
; GFX8-NEXT: [[FABS:%[0-9]+]]:_(s16) = G_FABS [[FSUB]]
@@ -1108,12 +1113,13 @@ body: |
; GFX8-NEXT: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[INTRINSIC_TRUNC3]], [[OR3]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
; GFX8-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FADD3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C6]](s32)
; GFX8-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -1124,15 +1130,15 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s16) = G_INTRINSIC_TRUNC [[TRUNC]]
; GFX9-NEXT: [[FSUB:%[0-9]+]]:_(s16) = G_FSUB [[TRUNC]], [[INTRINSIC_TRUNC]]
; GFX9-NEXT: [[FABS:%[0-9]+]]:_(s16) = G_FABS [[FSUB]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir
index e306c50b07f17..95099b7998190 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir
@@ -165,11 +165,11 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; SI-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FPEXT]]
; SI-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[INTRINSIC_TRUNC]](s32)
@@ -178,7 +178,8 @@ body: |
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INTRINSIC_TRUNC1]](s32)
; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -187,11 +188,11 @@ body: |
; CI: liveins: $vgpr0
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; CI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
; CI-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FPEXT]]
; CI-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[INTRINSIC_TRUNC]](s32)
@@ -200,7 +201,8 @@ body: |
; CI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INTRINSIC_TRUNC1]](s32)
; CI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -209,16 +211,17 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s16) = G_INTRINSIC_TRUNC [[TRUNC]]
; VI-NEXT: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s16) = G_INTRINSIC_TRUNC [[TRUNC1]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INTRINSIC_TRUNC]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INTRINSIC_TRUNC1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -227,11 +230,11 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s16) = G_INTRINSIC_TRUNC [[TRUNC]]
; GFX9-NEXT: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s16) = G_INTRINSIC_TRUNC [[TRUNC1]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[INTRINSIC_TRUNC]](s16), [[INTRINSIC_TRUNC1]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
index cd67cbd27f38f..0dd90704109f7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
@@ -113,10 +113,10 @@ define amdgpu_ps void @image_store_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr2
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[COPY10]](<2 x f16>)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x i16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[BITCAST1]](s32), [[LSHR]](s32)
+ ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x i16>)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[BITCAST1]](i32), [[LSHR]](i32)
; UNPACKED-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<2 x i32>), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (<2 x f16>), addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
@@ -223,12 +223,12 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr3
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY11]](<2 x f16>)
+ ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY10]](<2 x f16>)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY11]](<2 x f16>)
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[BITCAST1]](s32)
+ ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[BITCAST1]](i32)
; UNPACKED-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<3 x i32>), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (<3 x f16>), align 8, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
@@ -249,22 +249,23 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
; GFX81-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX81-NEXT: [[COPY10:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr2
; GFX81-NEXT: [[COPY11:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr3
- ; GFX81-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x f16>)
- ; GFX81-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX81-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX81-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY11]](<2 x f16>)
+ ; GFX81-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY10]](<2 x f16>)
+ ; GFX81-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX81-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX81-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY11]](<2 x f16>)
; GFX81-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; GFX81-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX81-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; GFX81-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; GFX81-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX81-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
; GFX81-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX81-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX81-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; GFX81-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX81-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; GFX81-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX81-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
; GFX81-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; GFX81-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX81-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; GFX81-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL1]]
; GFX81-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX81-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x i16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
; GFX81-NEXT: [[BITCAST5:%[0-9]+]]:_(<3 x i32>) = G_BITCAST [[CONCAT_VECTORS]](<6 x i16>)
@@ -360,12 +361,12 @@ define amdgpu_ps void @image_store_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<4 x i16>) = G_BITCAST [[CONCAT_VECTORS]](<4 x f16>)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[BITCAST]](<4 x i16>)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; UNPACKED-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[BITCAST1]](s32), [[LSHR]](s32), [[BITCAST2]](s32), [[LSHR1]](s32)
+ ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; UNPACKED-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[BITCAST1]](i32), [[LSHR]](i32), [[BITCAST2]](i32), [[LSHR1]](i32)
; UNPACKED-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<4 x i32>), 15, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (<4 x f16>), addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
index 2ead01f20098d..1457569827549 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
@@ -6558,21 +6558,22 @@ body: |
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 4, align 4)
- ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; CI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -6585,21 +6586,22 @@ body: |
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 4, align 4)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -6720,21 +6722,22 @@ body: |
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 4, align 4)
- ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; CI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -6747,21 +6750,22 @@ body: |
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 4, align 4)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
index d79c310c197ab..eec162a7f106b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
@@ -9817,21 +9817,22 @@ body: |
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, align 4, addrspace 5)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -9844,21 +9845,22 @@ body: |
; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, align 4, addrspace 5)
- ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; CI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -9871,21 +9873,22 @@ body: |
; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, align 4, addrspace 5)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -9899,11 +9902,11 @@ body: |
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, align 4, addrspace 5)
; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
@@ -9920,11 +9923,11 @@ body: |
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, align 4, addrspace 5)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
@@ -9989,11 +9992,11 @@ body: |
; UNALIGNED_GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, align 4, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
- ; UNALIGNED_GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; UNALIGNED_GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; UNALIGNED_GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNALIGNED_GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; UNALIGNED_GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; UNALIGNED_GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; UNALIGNED_GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNALIGNED_GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; UNALIGNED_GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; UNALIGNED_GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; UNALIGNED_GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
; UNALIGNED_GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
@@ -10010,11 +10013,11 @@ body: |
; UNALIGNED_GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, align 4, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
- ; UNALIGNED_GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; UNALIGNED_GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; UNALIGNED_GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNALIGNED_GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; UNALIGNED_GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; UNALIGNED_GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; UNALIGNED_GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNALIGNED_GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; UNALIGNED_GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; UNALIGNED_GFX10-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; UNALIGNED_GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
; UNALIGNED_GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
@@ -10156,11 +10159,11 @@ body: |
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
@@ -10177,11 +10180,11 @@ body: |
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
@@ -10495,11 +10498,11 @@ body: |
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, align 1, addrspace 5)
; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
@@ -10516,11 +10519,11 @@ body: |
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, align 1, addrspace 5)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
- ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
index 2da9ab7fea77d..5cd202f3cdb92 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
@@ -560,19 +560,20 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[LSHR1]](s32)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[LSHR1]](i32)
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C1]]
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -582,20 +583,21 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC2]](s16)
; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[TRUNC3]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR2]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR3]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -624,9 +626,9 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
@@ -634,7 +636,8 @@ body: |
; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[UV1]](s32)
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C1]]
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -644,11 +647,11 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC2]](s16)
@@ -656,7 +659,8 @@ body: |
; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[TRUNC3]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR1]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR2]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -666,11 +670,11 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC2]](s16)
@@ -696,30 +700,31 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[LSHR1]](s32)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[LSHR1]](i32)
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C1]]
; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -731,31 +736,32 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC3]](s16)
; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[TRUNC4]](s16)
; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[TRUNC5]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR2]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR3]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR4]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -767,20 +773,20 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX9-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; GFX9-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UV4]], [[UV6]](<2 x s16>)
; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC1]](s16)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[LSHR]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[LSHR]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[LSHR1]](s16), [[DEF]](s16)
@@ -807,34 +813,35 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; SI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[LSHR1]](s32)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[LSHR1]](i32)
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C1]]
; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL2]]
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -845,35 +852,36 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; VI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC3]](s16)
; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[TRUNC4]](s16)
; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[TRUNC5]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR2]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR3]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR4]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL2]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL2]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -884,20 +892,20 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX9-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
; GFX9-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: [[UV9:%[0-9]+]]:_(<2 x s16>), [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UV3]], [[UV9]](<2 x s16>)
; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC1]](s16)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[LSHR]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[LSHR]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[LSHR1]](s16), [[DEF]](s16)
@@ -926,33 +934,34 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
- ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[LSHR2]](s32)
+ ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[LSHR2]](i32)
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
- ; SI-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[LSHR1]], [[LSHR3]](s32)
+ ; SI-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[LSHR1]], [[LSHR3]](i32)
; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C1]]
; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C1]]
; SI-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR7]], [[C1]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -964,36 +973,37 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC4]](s16)
; VI-NEXT: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[TRUNC5]](s16)
; VI-NEXT: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[TRUNC6]](s16)
; VI-NEXT: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[TRUNC7]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR4]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR5]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR6]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR7]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-mul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-mul.mir
index 2d39afc896dab..f7f1aa6ef3409 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-mul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-mul.mir
@@ -272,17 +272,18 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[BITCAST]], [[BITCAST1]]
; GFX6-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[LSHR1]]
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C1]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -292,20 +293,21 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[MUL:%[0-9]+]]:_(s16) = G_MUL [[TRUNC]], [[TRUNC2]]
; GFX8-NEXT: [[MUL1:%[0-9]+]]:_(s16) = G_MUL [[TRUNC1]], [[TRUNC3]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[MUL]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[MUL1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -387,13 +389,13 @@ body: |
; GFX9PLUS-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC5]](s16), [[DEF]](s16)
; GFX9PLUS-NEXT: [[MUL:%[0-9]+]]:_(<2 x s16>) = G_MUL [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
; GFX9PLUS-NEXT: [[MUL1:%[0-9]+]]:_(<2 x s16>) = G_MUL [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
- ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[MUL]](<2 x s16>)
- ; GFX9PLUS-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9PLUS-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9PLUS-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[MUL1]](<2 x s16>)
- ; GFX9PLUS-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[MUL]](<2 x s16>)
+ ; GFX9PLUS-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9PLUS-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9PLUS-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[MUL1]](<2 x s16>)
+ ; GFX9PLUS-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9PLUS-NEXT: S_ENDPGM 0, implicit [[TRUNC6]](s16), implicit [[TRUNC7]](s16), implicit [[TRUNC8]](s16)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
@@ -426,16 +428,16 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; GFX6-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[BITCAST]], [[BITCAST2]]
; GFX6-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[LSHR2]]
; GFX6-NEXT: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[BITCAST1]], [[BITCAST3]]
@@ -443,12 +445,13 @@ body: |
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C1]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL2]], [[C1]]
; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[MUL3]], [[C1]]
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -460,36 +463,37 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX8-NEXT: [[MUL:%[0-9]+]]:_(s16) = G_MUL [[TRUNC]], [[TRUNC4]]
; GFX8-NEXT: [[MUL1:%[0-9]+]]:_(s16) = G_MUL [[TRUNC1]], [[TRUNC5]]
; GFX8-NEXT: [[MUL2:%[0-9]+]]:_(s16) = G_MUL [[TRUNC2]], [[TRUNC6]]
; GFX8-NEXT: [[MUL3:%[0-9]+]]:_(s16) = G_MUL [[TRUNC3]], [[TRUNC7]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[MUL]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[MUL1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[MUL2]](s16)
; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[MUL3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
index 3f56e8e66c0c1..c7a0eba2336de 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
@@ -517,9 +517,10 @@ body: |
; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(<4 x s16>) = G_OR [[CONCAT_VECTORS]], [[CONCAT_VECTORS1]]
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[OR4]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C2]](i32)
+ ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL4]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
@@ -536,15 +537,15 @@ body: |
; CHECK-NEXT: [[CONCAT_VECTORS3:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST8]](<2 x s16>), [[BITCAST9]](<2 x s16>)
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(<4 x s16>) = G_OR [[CONCAT_VECTORS2]], [[CONCAT_VECTORS3]]
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[OR9]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST10]], [[C1]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST4]], [[C2]]
+ ; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C2]](i32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST4]], [[C3]]
; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
; CHECK-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL8]]
; CHECK-NEXT: [[BITCAST11:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR10]](i32)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST5]], [[C2]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST10]], [[C2]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST5]], [[C3]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST10]], [[C3]]
; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
; CHECK-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL9]]
; CHECK-NEXT: [[BITCAST12:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR11]](i32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
index a3d8c6e92f5ca..6a80867cd9552 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
@@ -71,17 +71,18 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[BITCAST]], [[BITCAST1]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR1]]
; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: G_BR %bb.2
@@ -128,13 +129,14 @@ body: |
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C]]
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C3]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -147,16 +149,16 @@ body: |
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[BITCAST]], [[BITCAST]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR]]
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[BITCAST1]], [[BITCAST1]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C3]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C3]]
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C4]](s32)
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C4]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C4]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C5]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C3]]
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C4]]
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND3]], [[SHL2]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
@@ -165,10 +167,10 @@ body: |
; CHECK-NEXT: bb.2:
; CHECK-NEXT: [[PHI:%[0-9]+]]:_(<4 x s16>) = G_PHI [[CONCAT_VECTORS]](<4 x s16>), %bb.0, [[CONCAT_VECTORS1]](<4 x s16>), %bb.1
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[PHI]](<4 x s16>)
- ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C7]](s32)
- ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL3]]
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C7]], [[C8]](s32)
+ ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[C7]], [[SHL3]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV2]](<2 x s16>), [[UV3]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
@@ -221,15 +223,15 @@ body: |
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C1]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[BITCAST]], [[BITCAST2]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR2]]
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[BITCAST1]], [[BITCAST3]]
@@ -237,12 +239,13 @@ body: |
; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C2]]
; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD3]], [[C2]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -1621,16 +1624,16 @@ body: |
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C2]]
; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[AND]](s32), [[AND1]]
- ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[LSHR]](s32), [[LSHR1]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[LSHR]](i32), [[LSHR1]]
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY3]](s32), [[C]]
; CHECK-NEXT: G_BRCOND [[ICMP2]](s1), %bb.1
; CHECK-NEXT: G_BR %bb.2
@@ -1638,16 +1641,16 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C3]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C3]](s32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C3]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C3]](i32)
; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C4]]
; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C4]]
; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[AND2]](s32), [[AND3]]
- ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[LSHR2]](s32), [[LSHR3]]
+ ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[LSHR2]](i32), [[LSHR3]]
; CHECK-NEXT: G_BR %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddo.mir
index bafe4ef214017..0870b74b4cd3a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddo.mir
@@ -140,43 +140,44 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[BITCAST]], [[BITCAST1]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR1]]
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[ADD]], 16
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST3]], 16
; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG]](s32), [[SEXT_INREG1]]
; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[ADD1]], 16
; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR2]], 16
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG2]](s32), [[SEXT_INREG3]]
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
; CHECK-NEXT: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST4]], 16
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG4]](s32), [[COPY2]]
; CHECK-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR3]], 16
- ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG5]](s32), [[C2]]
+ ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG5]](s32), [[C3]]
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP2]], [[ICMP]]
; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP1]]
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR1]](s1)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY [[BITCAST2]](<2 x s16>)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND2]](s32), [[AND3]](s32)
; CHECK-NEXT: $vgpr0 = COPY [[COPY3]](<2 x s16>)
; CHECK-NEXT: $vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<2 x s32>)
@@ -199,21 +200,21 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[BITCAST]], [[BITCAST2]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR1]]
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[BITCAST1]], [[BITCAST3]]
; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV6]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV6]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[ADD]], 16
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST4]], 16
; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG]](s32), [[SEXT_INREG1]]
@@ -224,9 +225,9 @@ body: |
; CHECK-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST5]], 16
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG4]](s32), [[SEXT_INREG5]]
; CHECK-NEXT: [[UV9:%[0-9]+]]:_(<2 x s16>), [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[UV9]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[UV10]](<2 x s16>)
; CHECK-NEXT: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST6]], 16
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
@@ -245,21 +246,22 @@ body: |
; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C2]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C3]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C3]](s32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL2]]
; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST8]](<2 x s16>), [[BITCAST9]](<2 x s16>), [[BITCAST10]](<2 x s16>)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND3]](s32), [[AND4]](s32), [[AND5]](s32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
@@ -287,16 +289,16 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr1_vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[BITCAST]], [[BITCAST2]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR2]]
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[BITCAST1]], [[BITCAST3]]
@@ -304,20 +306,21 @@ body: |
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C1]]
; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD3]], [[C1]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; CHECK-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST7]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[ADD]], 16
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST6]], 16
; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG]](s32), [[SEXT_INREG1]]
@@ -331,22 +334,22 @@ body: |
; CHECK-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR5]], 16
; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG6]](s32), [[SEXT_INREG7]]
; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(s32) = G_BITCAST [[UV6]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST8]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST9]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(i32) = G_BITCAST [[UV6]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST8]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST9]], [[C]](i32)
; CHECK-NEXT: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST8]], 16
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG8]](s32), [[COPY2]]
; CHECK-NEXT: [[SEXT_INREG9:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR6]], 16
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG9]](s32), [[COPY3]]
; CHECK-NEXT: [[SEXT_INREG10:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST9]], 16
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG10]](s32), [[COPY4]]
; CHECK-NEXT: [[SEXT_INREG11:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR7]], 16
- ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG11]](s32), [[C2]]
+ ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG11]](s32), [[C3]]
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP4]], [[ICMP]]
; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP5]], [[ICMP1]]
; CHECK-NEXT: [[XOR2:%[0-9]+]]:_(s1) = G_XOR [[ICMP6]], [[ICMP2]]
@@ -356,11 +359,11 @@ body: |
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR2]](s1)
; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR3]](s1)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(<4 x s16>) = COPY [[CONCAT_VECTORS]](<4 x s16>)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C4]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[AND5]](s32), [[AND6]](s32), [[AND7]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[COPY5]](<4 x s16>)
; CHECK-NEXT: $vgpr2_vgpr3_vgpr4_vgpr5 = COPY [[BUILD_VECTOR]](<4 x s32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
index 6b302048ac7b5..0467a3918bfa5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
@@ -254,7 +254,6 @@ body: |
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
- ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
@@ -262,20 +261,21 @@ body: |
; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
- ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C2]](s16), [[C2]](s16)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[BUILD_VECTOR]], [[BUILD_VECTOR2]](<2 x s16>)
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[BUILD_VECTOR1]], [[BUILD_VECTOR2]](<2 x s16>)
; GFX9-NEXT: [[SADDSAT:%[0-9]+]]:_(<2 x s16>) = G_SADDSAT [[SHL]], [[SHL1]]
; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(<2 x s16>) = G_ASHR [[SADDSAT]], [[BUILD_VECTOR2]](<2 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[ASHR]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[ASHR]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C2]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
; GFX9-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C3]]
; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C3]]
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C2]](s16)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
; GFX9-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL2]]
; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
@@ -368,38 +368,39 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
- ; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX]]
- ; GFX6-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
- ; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN]]
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C4]]
+ ; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMAX]]
+ ; GFX6-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C4]]
+ ; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SMIN]]
; GFX6-NEXT: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[SHL1]]
; GFX6-NEXT: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
; GFX6-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[SMIN1]]
- ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
- ; GFX6-NEXT: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C3]]
- ; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX2]]
- ; GFX6-NEXT: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C3]]
- ; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN2]]
+ ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C4]]
+ ; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMAX2]]
+ ; GFX6-NEXT: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C4]]
+ ; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SMIN2]]
; GFX6-NEXT: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
; GFX6-NEXT: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB2]]
; GFX6-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[SMIN3]]
- ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ADD1]], [[C]](s32)
- ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C4]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C4]]
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ADD1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -409,15 +410,15 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
; GFX8-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
; GFX8-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
@@ -437,7 +438,8 @@ body: |
; GFX8-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[SMIN3]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -466,57 +468,58 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
- ; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX]]
- ; GFX6-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
- ; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN]]
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C4]]
+ ; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMAX]]
+ ; GFX6-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C4]]
+ ; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SMIN]]
; GFX6-NEXT: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[SHL1]]
; GFX6-NEXT: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
; GFX6-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[SMIN1]]
- ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C3]]
- ; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX2]]
- ; GFX6-NEXT: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C3]]
- ; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN2]]
+ ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C4]]
+ ; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMAX2]]
+ ; GFX6-NEXT: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C4]]
+ ; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SMIN2]]
; GFX6-NEXT: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
; GFX6-NEXT: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB2]]
; GFX6-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[SMIN3]]
- ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ADD1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C]](s32)
- ; GFX6-NEXT: [[SMAX4:%[0-9]+]]:_(s32) = G_SMAX [[SHL4]], [[C3]]
- ; GFX6-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX4]]
- ; GFX6-NEXT: [[SMIN4:%[0-9]+]]:_(s32) = G_SMIN [[SHL4]], [[C3]]
- ; GFX6-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN4]]
+ ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ADD1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SMAX4:%[0-9]+]]:_(s32) = G_SMAX [[SHL4]], [[C4]]
+ ; GFX6-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMAX4]]
+ ; GFX6-NEXT: [[SMIN4:%[0-9]+]]:_(s32) = G_SMIN [[SHL4]], [[C4]]
+ ; GFX6-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SMIN4]]
; GFX6-NEXT: [[SMAX5:%[0-9]+]]:_(s32) = G_SMAX [[SUB5]], [[SHL5]]
; GFX6-NEXT: [[SMIN5:%[0-9]+]]:_(s32) = G_SMIN [[SMAX5]], [[SUB4]]
; GFX6-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SHL4]], [[SMIN5]]
- ; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[ADD2]], [[C]](s32)
- ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C4]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C4]]
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[ADD2]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL6]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C4]]
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C5]]
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL7]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL7]]
+ ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL7]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -526,19 +529,19 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
; GFX8-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
; GFX8-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
@@ -565,15 +568,16 @@ body: |
; GFX8-NEXT: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[SMIN5]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C]](s32)
+ ; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL1]]
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -583,25 +587,25 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
; GFX9-NEXT: [[SADDSAT:%[0-9]+]]:_(<2 x s16>) = G_SADDSAT [[UV]], [[BUILD_VECTOR]]
; GFX9-NEXT: [[SADDSAT1:%[0-9]+]]:_(<2 x s16>) = G_SADDSAT [[UV1]], [[BUILD_VECTOR1]]
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[SADDSAT]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[SADDSAT1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[SADDSAT]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[SADDSAT1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC3]](s16), [[TRUNC4]](s16)
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC5]](s16), [[DEF]](s16)
; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
@@ -627,68 +631,69 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
- ; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX]]
- ; GFX6-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
- ; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN]]
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C1]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C4]]
+ ; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMAX]]
+ ; GFX6-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C4]]
+ ; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SMIN]]
; GFX6-NEXT: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[SHL1]]
; GFX6-NEXT: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
; GFX6-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[SMIN1]]
- ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C]](s32)
- ; GFX6-NEXT: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C3]]
- ; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX2]]
- ; GFX6-NEXT: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C3]]
- ; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN2]]
+ ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C4]]
+ ; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMAX2]]
+ ; GFX6-NEXT: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C4]]
+ ; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SMIN2]]
; GFX6-NEXT: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
; GFX6-NEXT: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB2]]
; GFX6-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[SMIN3]]
- ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ADD1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[BITCAST3]], [[C]](s32)
- ; GFX6-NEXT: [[SMAX4:%[0-9]+]]:_(s32) = G_SMAX [[SHL4]], [[C3]]
- ; GFX6-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX4]]
- ; GFX6-NEXT: [[SMIN4:%[0-9]+]]:_(s32) = G_SMIN [[SHL4]], [[C3]]
- ; GFX6-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN4]]
+ ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ADD1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[BITCAST3]], [[C1]](s32)
+ ; GFX6-NEXT: [[SMAX4:%[0-9]+]]:_(s32) = G_SMAX [[SHL4]], [[C4]]
+ ; GFX6-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMAX4]]
+ ; GFX6-NEXT: [[SMIN4:%[0-9]+]]:_(s32) = G_SMIN [[SHL4]], [[C4]]
+ ; GFX6-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SMIN4]]
; GFX6-NEXT: [[SMAX5:%[0-9]+]]:_(s32) = G_SMAX [[SUB5]], [[SHL5]]
; GFX6-NEXT: [[SMIN5:%[0-9]+]]:_(s32) = G_SMIN [[SMAX5]], [[SUB4]]
; GFX6-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SHL4]], [[SMIN5]]
- ; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[ADD2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C]](s32)
- ; GFX6-NEXT: [[SMAX6:%[0-9]+]]:_(s32) = G_SMAX [[SHL6]], [[C3]]
- ; GFX6-NEXT: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX6]]
- ; GFX6-NEXT: [[SMIN6:%[0-9]+]]:_(s32) = G_SMIN [[SHL6]], [[C3]]
- ; GFX6-NEXT: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN6]]
+ ; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[ADD2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C1]](s32)
+ ; GFX6-NEXT: [[SMAX6:%[0-9]+]]:_(s32) = G_SMAX [[SHL6]], [[C4]]
+ ; GFX6-NEXT: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMAX6]]
+ ; GFX6-NEXT: [[SMIN6:%[0-9]+]]:_(s32) = G_SMIN [[SHL6]], [[C4]]
+ ; GFX6-NEXT: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SMIN6]]
; GFX6-NEXT: [[SMAX7:%[0-9]+]]:_(s32) = G_SMAX [[SUB7]], [[SHL7]]
; GFX6-NEXT: [[SMIN7:%[0-9]+]]:_(s32) = G_SMIN [[SMAX7]], [[SUB6]]
; GFX6-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SHL6]], [[SMIN7]]
- ; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[ADD3]], [[C]](s32)
- ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C4]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C4]]
- ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[ADD3]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL8]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C4]]
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C4]]
- ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C5]]
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C5]]
+ ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL9]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -700,24 +705,24 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
; GFX8-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
; GFX8-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
@@ -751,12 +756,13 @@ body: |
; GFX8-NEXT: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[TRUNC3]], [[SMIN7]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C4]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sdiv.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sdiv.mir
index 673478bddf875..e97c7beab2425 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sdiv.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sdiv.mir
@@ -2702,11 +2702,11 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; GFX6-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 31
@@ -2774,7 +2774,8 @@ body: |
; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB3]], [[C5]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB7]], [[C5]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -2784,11 +2785,11 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX8-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; GFX8-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 31
@@ -2856,7 +2857,8 @@ body: |
; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB3]], [[C5]]
; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB7]], [[C5]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX8-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -2866,11 +2868,11 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX9-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; GFX9-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 31
@@ -2945,11 +2947,11 @@ body: |
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX10-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; GFX10-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 31
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
index 69d5988528296..0fae63e3ad1d6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
@@ -1314,35 +1314,36 @@ body: |
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](s32), [[UV4]]
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV2]](s32), [[UV5]]
; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV6]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV6]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; CHECK-NEXT: [[UV9:%[0-9]+]]:_(<2 x s16>), [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
- ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV9]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV10]](<2 x s16>)
+ ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s16) = G_SELECT [[ICMP]](s1), [[TRUNC]], [[TRUNC3]]
; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s16) = G_SELECT [[ICMP1]](s1), [[TRUNC1]], [[TRUNC4]]
; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s16) = G_SELECT [[ICMP2]](s1), [[TRUNC2]], [[TRUNC5]]
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT]](s16)
; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT2]](s16)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL2]]
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL2]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext-inreg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext-inreg.mir
index b4ce1b2c06ac0..ac9a64ff5b20f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext-inreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext-inreg.mir
@@ -880,15 +880,16 @@ body: |
; GFX8: liveins: $vgpr0
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX8-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 1
; GFX8-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 1
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C1]]
; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C1]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -897,15 +898,16 @@ body: |
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX6-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 1
; GFX6-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 1
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C1]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -933,13 +935,13 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16)
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[UV1]], [[BUILD_VECTOR1]](<2 x s16>)
; GFX9-NEXT: [[ASHR1:%[0-9]+]]:_(<2 x s16>) = G_ASHR [[SHL1]], [[BUILD_VECTOR1]](<2 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[ASHR]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[ASHR1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[ASHR]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[ASHR1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
@@ -951,26 +953,27 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; GFX8-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 1
; GFX8-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 1
; GFX8-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 1
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C1]]
; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C1]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL2]]
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -980,26 +983,27 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; GFX6-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 1
; GFX6-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 1
; GFX6-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 1
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C1]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C1]]
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
- ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL2]]
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -1126,11 +1130,11 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX8-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 1
; GFX8-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 1
; GFX8-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 1
@@ -1138,12 +1142,13 @@ body: |
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C1]]
; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C1]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C1]]
; GFX8-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG3]], [[C1]]
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -1154,11 +1159,11 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 1
; GFX6-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 1
; GFX6-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 1
@@ -1166,12 +1171,13 @@ body: |
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C1]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C1]]
; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG3]], [[C1]]
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext.mir
index 1f34bd78639db..52e47907b8688 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext.mir
@@ -101,9 +101,9 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG]](s32), [[SEXT_INREG1]](s32)
@@ -124,10 +124,10 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
@@ -150,11 +150,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
index 4e4e79cf08b0f..7a10879c955a8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
@@ -553,18 +553,19 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[AND]](s32)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[LSHR1]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[LSHR1]](i32)
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C1]]
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL2]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -574,20 +575,21 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC2]](s16)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC3]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SHL]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SHL1]](s16)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -616,16 +618,17 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[UV]](s32)
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[UV1]](s32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C1]]
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -635,11 +638,11 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC2]](s16)
@@ -647,7 +650,8 @@ body: |
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC3]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SHL]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SHL1]](s16)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -657,11 +661,11 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC2]](s16)
@@ -686,32 +690,33 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; SI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[AND]](s32)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[LSHR1]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[LSHR1]](i32)
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[AND1]](s32)
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C1]]
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL3]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL4]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL5]]
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL5]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -722,35 +727,36 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; VI-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC3]](s16)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC4]](s16)
; VI-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[TRUNC5]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SHL]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SHL1]](s16)
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SHL2]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL5]]
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL5]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -761,20 +767,20 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX9-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
; GFX9-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: [[UV9:%[0-9]+]]:_(<2 x s16>), [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[UV3]], [[UV9]](<2 x s16>)
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[SHL]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[SHL]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[SHL1]](s16), [[DEF]](s16)
@@ -804,31 +810,32 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[AND]](s32)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[LSHR2]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[LSHR2]](i32)
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[AND1]](s32)
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[LSHR3]](s32)
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[LSHR3]](i32)
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C1]]
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C1]]
; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C1]]
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C]](s32)
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL5]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -840,36 +847,37 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC4]](s16)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC5]](s16)
; VI-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[TRUNC6]](s16)
; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[TRUNC7]](s16)
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SHL]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SHL1]](s16)
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SHL2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SHL3]](s16)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir
index 129f727fc922e..2637115731133 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir
@@ -300,20 +300,21 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir
index fb01c1402a218..d4797f698a7a8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir
@@ -943,20 +943,21 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
@@ -968,16 +969,16 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC3]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC]](s16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir
index 165be3a148379..dc0fd643dae49 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir
@@ -402,11 +402,11 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; SI-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; SI-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SEXT_INREG]], [[SEXT_INREG1]]
@@ -416,7 +416,8 @@ body: |
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SMAX]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SMAX1]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -426,20 +427,21 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC1]], [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SMAX]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SMAX1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -502,11 +504,11 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX9-NEXT: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
; GFX9-NEXT: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[DEF]], [[DEF]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[SMAX]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[SMAX]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SMAX1]](s16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[ANYEXT]](s32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[ANYEXT]](s32)
; GFX9-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x s32>)
;
; GFX1250-LABEL: name: test_smax_v3s16
@@ -517,11 +519,11 @@ body: |
; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX1250-NEXT: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
; GFX1250-NEXT: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[DEF]], [[DEF]]
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[SMAX]](<2 x s16>)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[SMAX]](<2 x s16>)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SMAX1]](s16)
- ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[ANYEXT]](s32)
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[ANYEXT]](s32)
; GFX1250-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x s32>)
%0:_(<3 x s16>) = G_IMPLICIT_DEF
%1:_(<3 x s16>) = G_IMPLICIT_DEF
@@ -542,16 +544,16 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; SI-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; SI-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST2]], 16
; SI-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SEXT_INREG]], [[SEXT_INREG1]]
@@ -567,12 +569,13 @@ body: |
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SMAX]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SMAX1]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SMAX2]], [[C1]]
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SMAX3]], [[C1]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -584,36 +587,37 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[TRUNC4]]
; VI-NEXT: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC2]], [[TRUNC6]]
; VI-NEXT: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC3]], [[TRUNC7]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SMAX]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SMAX1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SMAX2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SMAX3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir
index ebf1429cf9283..78e790d27616c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir
@@ -402,11 +402,11 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; SI-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; SI-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SEXT_INREG]], [[SEXT_INREG1]]
@@ -416,7 +416,8 @@ body: |
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SMIN]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SMIN1]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -426,20 +427,21 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC1]], [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SMIN]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SMIN1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -502,11 +504,11 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX9-NEXT: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
; GFX9-NEXT: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[DEF]], [[DEF]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[SMIN]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[SMIN]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SMIN1]](s16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[ANYEXT]](s32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[ANYEXT]](s32)
; GFX9-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x s32>)
;
; GFX1250-LABEL: name: test_smin_v3s16
@@ -517,11 +519,11 @@ body: |
; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX1250-NEXT: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
; GFX1250-NEXT: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[DEF]], [[DEF]]
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[SMIN]](<2 x s16>)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[SMIN]](<2 x s16>)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SMIN1]](s16)
- ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[ANYEXT]](s32)
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[ANYEXT]](s32)
; GFX1250-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x s32>)
%0:_(<3 x s16>) = G_IMPLICIT_DEF
%1:_(<3 x s16>) = G_IMPLICIT_DEF
@@ -542,16 +544,16 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; SI-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; SI-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST2]], 16
; SI-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SEXT_INREG]], [[SEXT_INREG1]]
@@ -567,12 +569,13 @@ body: |
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SMIN]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SMIN1]], [[C1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SMIN2]], [[C1]]
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SMIN3]], [[C1]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -584,36 +587,37 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[TRUNC4]]
; VI-NEXT: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC2]], [[TRUNC6]]
; VI-NEXT: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC3]], [[TRUNC7]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SMIN]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SMIN1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SMIN2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SMIN3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulh.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulh.mir
index 9e1318a465cfe..11190df230991 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulh.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulh.mir
@@ -175,8 +175,9 @@ body: |
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C2]](i32)
; GFX8-NEXT: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; GFX8-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG4]](s32), [[SEXT_INREG5]](s32)
@@ -263,11 +264,11 @@ body: |
; GFX9-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16)
; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(<2 x s16>) = G_ASHR [[MUL]], [[BUILD_VECTOR2]](<2 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[ASHR]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[ASHR]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
; GFX9-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C2]]
; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C2]]
@@ -395,19 +396,20 @@ body: |
; GFX9-NEXT: [[MUL1:%[0-9]+]]:_(<2 x s16>) = G_MUL [[BUILD_VECTOR3]], [[BUILD_VECTOR4]]
; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C3]](s16), [[C3]](s16)
; GFX9-NEXT: [[ASHR1:%[0-9]+]]:_(<2 x s16>) = G_ASHR [[MUL1]], [[BUILD_VECTOR5]](<2 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[ASHR]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[ASHR1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; GFX9-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C4]]
- ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C4]]
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[ASHR]](<2 x s16>)
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C4]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[ASHR1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C4]](i32)
+ ; GFX9-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C5]]
+ ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C5]]
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C4]]
+ ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C5]]
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR7]], [[C4]]
+ ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR7]], [[C5]]
; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulo.mir
index 3ea3227115b47..3aee04dbed8e8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulo.mir
@@ -232,8 +232,9 @@ body: |
; GFX8-NEXT: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[ANYEXT]], 1
; GFX8-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[ANYEXT1]], 1
; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG6]](s32), [[SEXT_INREG7]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C2]](i32)
; GFX8-NEXT: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; GFX8-NEXT: [[SEXT_INREG9:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; GFX8-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG8]](s32), [[SEXT_INREG9]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-srem.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-srem.mir
index 11a56af2ce476..e062b7faf3b0c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-srem.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-srem.mir
@@ -2594,11 +2594,11 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; GFX6-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 31
@@ -2659,7 +2659,8 @@ body: |
; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB4]], [[C4]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB9]], [[C4]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -2669,11 +2670,11 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX8-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; GFX8-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 31
@@ -2734,7 +2735,8 @@ body: |
; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB4]], [[C4]]
; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB9]], [[C4]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -2744,11 +2746,11 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX9-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; GFX9-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 31
@@ -2816,11 +2818,11 @@ body: |
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX10-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 16
; GFX10-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 31
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sshlsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sshlsat.mir
index 7e80168072b88..5c8b097c19739 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sshlsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sshlsat.mir
@@ -398,35 +398,36 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C2]](s32)
; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[AND]](s32)
; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[AND]](s32)
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
- ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL]](s32), [[C4]]
- ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C2]], [[C3]]
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL]](s32), [[C5]]
+ ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C3]], [[C4]]
; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL]](s32), [[ASHR]]
; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SELECT]], [[SHL1]]
- ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SELECT1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[SHL2]], [[LSHR1]](s32)
- ; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL3]], [[LSHR1]](s32)
- ; GFX6-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL2]](s32), [[C4]]
- ; GFX6-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[C2]], [[C3]]
+ ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SELECT1]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[SHL2]], [[LSHR1]](i32)
+ ; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL3]], [[LSHR1]](i32)
+ ; GFX6-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL2]](s32), [[C5]]
+ ; GFX6-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[C3]], [[C4]]
; GFX6-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL2]](s32), [[ASHR2]]
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SELECT2]], [[SHL3]]
- ; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SELECT3]], [[C]](s32)
+ ; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SELECT3]], [[C2]](s32)
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C1]]
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL4]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -436,15 +437,15 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC2]](s16)
; GFX8-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SHL]], [[TRUNC2]](s16)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
@@ -462,7 +463,8 @@ body: |
; GFX8-NEXT: [[SELECT3:%[0-9]+]]:_(s16) = G_SELECT [[ICMP3]](s1), [[SELECT2]], [[SHL1]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT3]](s16)
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -472,15 +474,15 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC2]](s16)
; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SHL]], [[TRUNC2]](s16)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
@@ -515,52 +517,53 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[LSHR1]](s32)
- ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[LSHR1]](s32)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL]](s32), [[C3]]
- ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C1]], [[C2]]
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[LSHR1]](i32)
+ ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[LSHR1]](i32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL]](s32), [[C4]]
+ ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C2]], [[C3]]
; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL]](s32), [[ASHR]]
; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SELECT]], [[SHL1]]
- ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SELECT1]], [[C]](s32)
- ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C4]]
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SELECT1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C5]]
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[SHL2]], [[AND]](s32)
; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL3]], [[AND]](s32)
- ; GFX6-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL2]](s32), [[C3]]
- ; GFX6-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[C1]], [[C2]]
+ ; GFX6-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL2]](s32), [[C4]]
+ ; GFX6-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[C2]], [[C3]]
; GFX6-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL2]](s32), [[ASHR2]]
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SELECT2]], [[SHL3]]
- ; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SELECT3]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[SHL4]], [[LSHR2]](s32)
- ; GFX6-NEXT: [[ASHR4:%[0-9]+]]:_(s32) = G_ASHR [[SHL5]], [[LSHR2]](s32)
- ; GFX6-NEXT: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL4]](s32), [[C3]]
- ; GFX6-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP4]](s1), [[C1]], [[C2]]
+ ; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SELECT3]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[SHL4]], [[LSHR2]](i32)
+ ; GFX6-NEXT: [[ASHR4:%[0-9]+]]:_(s32) = G_ASHR [[SHL5]], [[LSHR2]](i32)
+ ; GFX6-NEXT: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL4]](s32), [[C4]]
+ ; GFX6-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP4]](s1), [[C2]], [[C3]]
; GFX6-NEXT: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL4]](s32), [[ASHR4]]
; GFX6-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT4]], [[SHL5]]
- ; GFX6-NEXT: [[ASHR5:%[0-9]+]]:_(s32) = G_ASHR [[SELECT5]], [[C]](s32)
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C4]]
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C4]]
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+ ; GFX6-NEXT: [[ASHR5:%[0-9]+]]:_(s32) = G_ASHR [[SELECT5]], [[C1]](s32)
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C5]]
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL6]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR5]], [[C4]]
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C]](s32)
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR5]], [[C5]]
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND3]], [[SHL7]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL7]]
+ ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL7]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -570,19 +573,19 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC3]](s16)
; GFX8-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SHL]], [[TRUNC3]](s16)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
@@ -606,15 +609,16 @@ body: |
; GFX8-NEXT: [[SELECT5:%[0-9]+]]:_(s16) = G_SELECT [[ICMP5]](s1), [[SELECT4]], [[SHL2]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT3]](s16)
- ; GFX8-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT5]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C]](s32)
+ ; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL4]]
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL4]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -624,19 +628,19 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC3]](s16)
; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SHL]], [[TRUNC3]](s16)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
@@ -684,62 +688,63 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C2]](s32)
; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[AND]](s32)
; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[AND]](s32)
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
- ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL]](s32), [[C4]]
- ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C2]], [[C3]]
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL]](s32), [[C5]]
+ ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C3]], [[C4]]
; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL]](s32), [[ASHR]]
; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SELECT]], [[SHL1]]
- ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SELECT1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[SHL2]], [[LSHR2]](s32)
- ; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL3]], [[LSHR2]](s32)
- ; GFX6-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL2]](s32), [[C4]]
- ; GFX6-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[C2]], [[C3]]
+ ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SELECT1]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[SHL2]], [[LSHR2]](i32)
+ ; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL3]], [[LSHR2]](i32)
+ ; GFX6-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL2]](s32), [[C5]]
+ ; GFX6-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[C3]], [[C4]]
; GFX6-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL2]](s32), [[ASHR2]]
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SELECT2]], [[SHL3]]
- ; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SELECT3]], [[C]](s32)
+ ; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SELECT3]], [[C2]](s32)
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C2]](s32)
; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[SHL4]], [[AND1]](s32)
; GFX6-NEXT: [[ASHR4:%[0-9]+]]:_(s32) = G_ASHR [[SHL5]], [[AND1]](s32)
- ; GFX6-NEXT: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL4]](s32), [[C4]]
- ; GFX6-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP4]](s1), [[C2]], [[C3]]
+ ; GFX6-NEXT: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL4]](s32), [[C5]]
+ ; GFX6-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP4]](s1), [[C3]], [[C4]]
; GFX6-NEXT: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL4]](s32), [[ASHR4]]
; GFX6-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT4]], [[SHL5]]
- ; GFX6-NEXT: [[ASHR5:%[0-9]+]]:_(s32) = G_ASHR [[SELECT5]], [[C]](s32)
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[SHL6]], [[LSHR3]](s32)
- ; GFX6-NEXT: [[ASHR6:%[0-9]+]]:_(s32) = G_ASHR [[SHL7]], [[LSHR3]](s32)
- ; GFX6-NEXT: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL6]](s32), [[C4]]
- ; GFX6-NEXT: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[ICMP6]](s1), [[C2]], [[C3]]
+ ; GFX6-NEXT: [[ASHR5:%[0-9]+]]:_(s32) = G_ASHR [[SELECT5]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[SHL6]], [[LSHR3]](i32)
+ ; GFX6-NEXT: [[ASHR6:%[0-9]+]]:_(s32) = G_ASHR [[SHL7]], [[LSHR3]](i32)
+ ; GFX6-NEXT: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SHL6]](s32), [[C5]]
+ ; GFX6-NEXT: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[ICMP6]](s1), [[C3]], [[C4]]
; GFX6-NEXT: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL6]](s32), [[ASHR6]]
; GFX6-NEXT: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP7]](s1), [[SELECT6]], [[SHL7]]
- ; GFX6-NEXT: [[ASHR7:%[0-9]+]]:_(s32) = G_ASHR [[SELECT7]], [[C]](s32)
+ ; GFX6-NEXT: [[ASHR7:%[0-9]+]]:_(s32) = G_ASHR [[SELECT7]], [[C2]](s32)
; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C1]]
- ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL8]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ASHR5]], [[C1]]
; GFX6-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ASHR7]], [[C1]]
- ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL9]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -751,24 +756,24 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC4]](s16)
; GFX8-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SHL]], [[TRUNC4]](s16)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
@@ -798,12 +803,13 @@ body: |
; GFX8-NEXT: [[SELECT7:%[0-9]+]]:_(s16) = G_SELECT [[ICMP7]](s1), [[SELECT6]], [[SHL3]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT3]](s16)
- ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT5]](s16)
; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT7]](s16)
- ; GFX8-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C4]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -815,24 +821,24 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC4]](s16)
; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SHL]], [[TRUNC4]](s16)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubo.mir
index 8767444283d8a..0b9293bae4409 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubo.mir
@@ -140,23 +140,24 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[BITCAST]], [[BITCAST1]]
; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[LSHR]], [[LSHR1]]
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST3]], 16
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST4]], 16
; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG]](s32), [[SEXT_INREG1]]
@@ -164,19 +165,19 @@ body: |
; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR3]], 16
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG2]](s32), [[SEXT_INREG3]]
; CHECK-NEXT: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SUB]], 16
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG4]](s32), [[COPY2]]
; CHECK-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SUB1]], 16
- ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG5]](s32), [[C2]]
+ ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG5]](s32), [[C3]]
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP]], [[ICMP2]]
; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP3]]
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR1]](s1)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY [[BITCAST2]](<2 x s16>)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND2]](s32), [[AND3]](s32)
; CHECK-NEXT: $vgpr0 = COPY [[COPY3]](<2 x s16>)
; CHECK-NEXT: $vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<2 x s32>)
@@ -199,25 +200,25 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[BITCAST]], [[BITCAST2]]
; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[LSHR]], [[LSHR1]]
; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[BITCAST1]], [[BITCAST3]]
; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV6]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV6]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
; CHECK-NEXT: [[UV9:%[0-9]+]]:_(<2 x s16>), [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[UV9]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[UV10]](<2 x s16>)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST4]], 16
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST6]], 16
; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG]](s32), [[SEXT_INREG1]]
@@ -245,21 +246,22 @@ body: |
; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C2]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB2]], [[C2]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C3]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C3]](s32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL2]]
; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST8]](<2 x s16>), [[BITCAST9]](<2 x s16>), [[BITCAST10]](<2 x s16>)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND3]](s32), [[AND4]](s32), [[AND5]](s32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
@@ -287,16 +289,16 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr1_vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[BITCAST]], [[BITCAST2]]
; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[LSHR]], [[LSHR2]]
; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[BITCAST1]], [[BITCAST3]]
@@ -304,25 +306,26 @@ body: |
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB2]], [[C1]]
; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB3]], [[C1]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; CHECK-NEXT: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST7]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(s32) = G_BITCAST [[UV6]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST8]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST9]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(i32) = G_BITCAST [[UV6]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST8]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST9]], [[C]](i32)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST6]], 16
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST8]], 16
; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG]](s32), [[SEXT_INREG1]]
@@ -336,17 +339,17 @@ body: |
; CHECK-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR7]], 16
; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG6]](s32), [[SEXT_INREG7]]
; CHECK-NEXT: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SUB]], 16
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG8]](s32), [[COPY2]]
; CHECK-NEXT: [[SEXT_INREG9:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SUB1]], 16
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG9]](s32), [[COPY3]]
; CHECK-NEXT: [[SEXT_INREG10:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SUB2]], 16
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG10]](s32), [[COPY4]]
; CHECK-NEXT: [[SEXT_INREG11:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SUB3]], 16
- ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG11]](s32), [[C2]]
+ ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG11]](s32), [[C3]]
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP]], [[ICMP4]]
; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP5]]
; CHECK-NEXT: [[XOR2:%[0-9]+]]:_(s1) = G_XOR [[ICMP2]], [[ICMP6]]
@@ -356,11 +359,11 @@ body: |
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR2]](s1)
; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR3]](s1)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(<4 x s16>) = COPY [[CONCAT_VECTORS]](<4 x s16>)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C4]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[AND5]](s32), [[AND6]](s32), [[AND7]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[COPY5]](<4 x s16>)
; CHECK-NEXT: $vgpr2_vgpr3_vgpr4_vgpr5 = COPY [[BUILD_VECTOR]](<4 x s32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
index a2ed479b3a865..9fc0fd8261c14 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
@@ -254,7 +254,6 @@ body: |
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
- ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
@@ -262,20 +261,21 @@ body: |
; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
- ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C2]](s16), [[C2]](s16)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[BUILD_VECTOR]], [[BUILD_VECTOR2]](<2 x s16>)
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[BUILD_VECTOR1]], [[BUILD_VECTOR2]](<2 x s16>)
; GFX9-NEXT: [[SSUBSAT:%[0-9]+]]:_(<2 x s16>) = G_SSUBSAT [[SHL]], [[SHL1]]
; GFX9-NEXT: [[ASHR:%[0-9]+]]:_(<2 x s16>) = G_ASHR [[SSUBSAT]], [[BUILD_VECTOR2]](<2 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[ASHR]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[ASHR]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C2]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
; GFX9-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C3]]
; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C3]]
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C2]](s16)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
; GFX9-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL2]]
; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
@@ -368,38 +368,39 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
- ; GFX6-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
- ; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C1]]
- ; GFX6-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
- ; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C2]]
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+ ; GFX6-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C4]]
+ ; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C2]]
+ ; GFX6-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C4]]
+ ; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C3]]
; GFX6-NEXT: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[SHL1]]
; GFX6-NEXT: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[SMIN1]]
- ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
- ; GFX6-NEXT: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C3]]
- ; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C1]]
- ; GFX6-NEXT: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C3]]
- ; GFX6-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C2]]
+ ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C4]]
+ ; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C2]]
+ ; GFX6-NEXT: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C4]]
+ ; GFX6-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C3]]
; GFX6-NEXT: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
; GFX6-NEXT: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB4]]
; GFX6-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[SMIN3]]
- ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SUB5]], [[C]](s32)
- ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C4]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C4]]
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SUB5]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -409,15 +410,15 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
; GFX8-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
; GFX8-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -437,7 +438,8 @@ body: |
; GFX8-NEXT: [[SUB5:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[SMIN3]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB5]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -466,58 +468,59 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
- ; GFX6-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
- ; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C1]]
- ; GFX6-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
- ; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C2]]
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+ ; GFX6-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C4]]
+ ; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C2]]
+ ; GFX6-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C4]]
+ ; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C3]]
; GFX6-NEXT: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[SHL1]]
; GFX6-NEXT: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[SMIN1]]
- ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C3]]
- ; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C1]]
- ; GFX6-NEXT: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C3]]
- ; GFX6-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C2]]
+ ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C4]]
+ ; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C2]]
+ ; GFX6-NEXT: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C4]]
+ ; GFX6-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C3]]
; GFX6-NEXT: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
; GFX6-NEXT: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB4]]
; GFX6-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[SMIN3]]
- ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SUB5]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C]](s32)
- ; GFX6-NEXT: [[SMAX4:%[0-9]+]]:_(s32) = G_SMAX [[SHL4]], [[C3]]
- ; GFX6-NEXT: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SMAX4]], [[C1]]
- ; GFX6-NEXT: [[SMIN4:%[0-9]+]]:_(s32) = G_SMIN [[SHL4]], [[C3]]
- ; GFX6-NEXT: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SMIN4]], [[C2]]
+ ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SUB5]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SMAX4:%[0-9]+]]:_(s32) = G_SMAX [[SHL4]], [[C4]]
+ ; GFX6-NEXT: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SMAX4]], [[C2]]
+ ; GFX6-NEXT: [[SMIN4:%[0-9]+]]:_(s32) = G_SMIN [[SHL4]], [[C4]]
+ ; GFX6-NEXT: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SMIN4]], [[C3]]
; GFX6-NEXT: [[SMAX5:%[0-9]+]]:_(s32) = G_SMAX [[SUB6]], [[SHL5]]
; GFX6-NEXT: [[SMIN5:%[0-9]+]]:_(s32) = G_SMIN [[SMAX5]], [[SUB7]]
; GFX6-NEXT: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SHL4]], [[SMIN5]]
- ; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SUB8]], [[C]](s32)
- ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C4]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C4]]
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SUB8]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL6]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C4]]
- ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C5]]
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL7]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL7]]
+ ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL7]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -527,19 +530,19 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
; GFX8-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
; GFX8-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -566,15 +569,16 @@ body: |
; GFX8-NEXT: [[SUB8:%[0-9]+]]:_(s16) = G_SUB [[TRUNC2]], [[SMIN5]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB5]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SUB8]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C]](s32)
+ ; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL1]]
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -584,25 +588,25 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
; GFX9-NEXT: [[SSUBSAT:%[0-9]+]]:_(<2 x s16>) = G_SSUBSAT [[UV]], [[BUILD_VECTOR]]
; GFX9-NEXT: [[SSUBSAT1:%[0-9]+]]:_(<2 x s16>) = G_SSUBSAT [[UV1]], [[BUILD_VECTOR1]]
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[SSUBSAT]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[SSUBSAT1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[SSUBSAT]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[SSUBSAT1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC3]](s16), [[TRUNC4]](s16)
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC5]](s16), [[DEF]](s16)
; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
@@ -628,68 +632,69 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
- ; GFX6-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
- ; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C1]]
- ; GFX6-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
- ; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C2]]
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C1]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+ ; GFX6-NEXT: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C4]]
+ ; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C2]]
+ ; GFX6-NEXT: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C4]]
+ ; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C3]]
; GFX6-NEXT: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[SHL1]]
; GFX6-NEXT: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[SMIN1]]
- ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C]](s32)
- ; GFX6-NEXT: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C3]]
- ; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C1]]
- ; GFX6-NEXT: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C3]]
- ; GFX6-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C2]]
+ ; GFX6-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C4]]
+ ; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C2]]
+ ; GFX6-NEXT: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C4]]
+ ; GFX6-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C3]]
; GFX6-NEXT: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
; GFX6-NEXT: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB4]]
; GFX6-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[SMIN3]]
- ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SUB5]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[BITCAST3]], [[C]](s32)
- ; GFX6-NEXT: [[SMAX4:%[0-9]+]]:_(s32) = G_SMAX [[SHL4]], [[C3]]
- ; GFX6-NEXT: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SMAX4]], [[C1]]
- ; GFX6-NEXT: [[SMIN4:%[0-9]+]]:_(s32) = G_SMIN [[SHL4]], [[C3]]
- ; GFX6-NEXT: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SMIN4]], [[C2]]
+ ; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SUB5]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[BITCAST3]], [[C1]](s32)
+ ; GFX6-NEXT: [[SMAX4:%[0-9]+]]:_(s32) = G_SMAX [[SHL4]], [[C4]]
+ ; GFX6-NEXT: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SMAX4]], [[C2]]
+ ; GFX6-NEXT: [[SMIN4:%[0-9]+]]:_(s32) = G_SMIN [[SHL4]], [[C4]]
+ ; GFX6-NEXT: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SMIN4]], [[C3]]
; GFX6-NEXT: [[SMAX5:%[0-9]+]]:_(s32) = G_SMAX [[SUB6]], [[SHL5]]
; GFX6-NEXT: [[SMIN5:%[0-9]+]]:_(s32) = G_SMIN [[SMAX5]], [[SUB7]]
; GFX6-NEXT: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SHL4]], [[SMIN5]]
- ; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SUB8]], [[C]](s32)
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C]](s32)
- ; GFX6-NEXT: [[SMAX6:%[0-9]+]]:_(s32) = G_SMAX [[SHL6]], [[C3]]
- ; GFX6-NEXT: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[SMAX6]], [[C1]]
- ; GFX6-NEXT: [[SMIN6:%[0-9]+]]:_(s32) = G_SMIN [[SHL6]], [[C3]]
- ; GFX6-NEXT: [[SUB10:%[0-9]+]]:_(s32) = G_SUB [[SMIN6]], [[C2]]
+ ; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SUB8]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C1]](s32)
+ ; GFX6-NEXT: [[SMAX6:%[0-9]+]]:_(s32) = G_SMAX [[SHL6]], [[C4]]
+ ; GFX6-NEXT: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[SMAX6]], [[C2]]
+ ; GFX6-NEXT: [[SMIN6:%[0-9]+]]:_(s32) = G_SMIN [[SHL6]], [[C4]]
+ ; GFX6-NEXT: [[SUB10:%[0-9]+]]:_(s32) = G_SUB [[SMIN6]], [[C3]]
; GFX6-NEXT: [[SMAX7:%[0-9]+]]:_(s32) = G_SMAX [[SUB9]], [[SHL7]]
; GFX6-NEXT: [[SMIN7:%[0-9]+]]:_(s32) = G_SMIN [[SMAX7]], [[SUB10]]
; GFX6-NEXT: [[SUB11:%[0-9]+]]:_(s32) = G_SUB [[SHL6]], [[SMIN7]]
- ; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SUB11]], [[C]](s32)
- ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C4]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C4]]
- ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SUB11]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL8]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C4]]
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C4]]
- ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C5]]
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C5]]
+ ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL9]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -701,24 +706,24 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
; GFX8-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
; GFX8-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -752,12 +757,13 @@ body: |
; GFX8-NEXT: [[SUB11:%[0-9]+]]:_(s16) = G_SUB [[TRUNC3]], [[SMIN7]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB5]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SUB8]](s16)
; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SUB11]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C4]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir
index 7a9868d12025c..5b97da12b16e2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir
@@ -3431,18 +3431,18 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C2]]
; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[AND]], [[C1]](i32)
; SI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
- ; SI-NEXT: G_STORE [[BITCAST]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+ ; SI-NEXT: G_STORE [[BITCAST]](i32), [[COPY]](p1) :: (store (s8), addrspace 1)
; SI-NEXT: G_STORE [[LSHR2]](i32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
; SI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
@@ -3450,7 +3450,7 @@ body: |
; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[AND1]], [[COPY2]](i32)
; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C3]](i64)
- ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
+ ; SI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
; SI-NEXT: G_STORE [[LSHR3]](i32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
; SI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; SI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
@@ -3458,7 +3458,7 @@ body: |
; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[BITCAST1]], [[C2]]
; SI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[AND2]], [[COPY3]](i32)
; SI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C3]](i64)
- ; SI-NEXT: G_STORE [[BITCAST1]](s32), [[PTR_ADD3]](p1) :: (store (s8) into unknown-address + 4, addrspace 1)
+ ; SI-NEXT: G_STORE [[BITCAST1]](i32), [[PTR_ADD3]](p1) :: (store (s8) into unknown-address + 4, addrspace 1)
; SI-NEXT: G_STORE [[LSHR4]](i32), [[PTR_ADD4]](p1) :: (store (s8) into unknown-address + 5, addrspace 1)
; SI-NEXT: [[C6:%[0-9]+]]:_(i64) = G_CONSTANT i64 6
; SI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C6]](i64)
@@ -3466,7 +3466,7 @@ body: |
; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
; SI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[AND3]], [[COPY4]](i32)
; SI-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C3]](i64)
- ; SI-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD5]](p1) :: (store (s8) into unknown-address + 6, addrspace 1)
+ ; SI-NEXT: G_STORE [[LSHR1]](i32), [[PTR_ADD5]](p1) :: (store (s8) into unknown-address + 6, addrspace 1)
; SI-NEXT: G_STORE [[LSHR5]](i32), [[PTR_ADD6]](p1) :: (store (s8) into unknown-address + 7, addrspace 1)
;
; CI-LABEL: name: test_store_global_v4s16_align1
@@ -3482,41 +3482,41 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(i16) = G_CONSTANT i16 8
; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i16) = G_LSHR [[TRUNC]], [[C1]](i16)
; VI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
- ; VI-NEXT: G_STORE [[BITCAST]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+ ; VI-NEXT: G_STORE [[BITCAST]](i32), [[COPY]](p1) :: (store (s8), addrspace 1)
; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[LSHR2]](i16)
; VI-NEXT: G_STORE [[ANYEXT]](i32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
; VI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i16) = G_LSHR [[TRUNC1]], [[C1]](i16)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C2]](i64)
- ; VI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
+ ; VI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
; VI-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[LSHR3]](i16)
; VI-NEXT: G_STORE [[ANYEXT1]](i32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; VI-NEXT: [[LSHR4:%[0-9]+]]:_(i16) = G_LSHR [[TRUNC2]], [[C1]](i16)
; VI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C2]](i64)
- ; VI-NEXT: G_STORE [[BITCAST1]](s32), [[PTR_ADD3]](p1) :: (store (s8) into unknown-address + 4, addrspace 1)
+ ; VI-NEXT: G_STORE [[BITCAST1]](i32), [[PTR_ADD3]](p1) :: (store (s8) into unknown-address + 4, addrspace 1)
; VI-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[LSHR4]](i16)
; VI-NEXT: G_STORE [[ANYEXT2]](i32), [[PTR_ADD4]](p1) :: (store (s8) into unknown-address + 5, addrspace 1)
; VI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 6
; VI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; VI-NEXT: [[LSHR5:%[0-9]+]]:_(i16) = G_LSHR [[TRUNC3]], [[C1]](i16)
; VI-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C2]](i64)
- ; VI-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD5]](p1) :: (store (s8) into unknown-address + 6, addrspace 1)
+ ; VI-NEXT: G_STORE [[LSHR1]](i32), [[PTR_ADD5]](p1) :: (store (s8) into unknown-address + 6, addrspace 1)
; VI-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[LSHR5]](i16)
; VI-NEXT: G_STORE [[ANYEXT3]](i32), [[PTR_ADD6]](p1) :: (store (s8) into unknown-address + 7, addrspace 1)
;
@@ -3557,21 +3557,21 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; SI-NEXT: G_STORE [[BITCAST]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: G_STORE [[BITCAST]](i32), [[COPY]](p1) :: (store (s16), addrspace 1)
; SI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
+ ; SI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
; SI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
- ; SI-NEXT: G_STORE [[BITCAST1]](s32), [[PTR_ADD1]](p1) :: (store (s16) into unknown-address + 4, addrspace 1)
+ ; SI-NEXT: G_STORE [[BITCAST1]](i32), [[PTR_ADD1]](p1) :: (store (s16) into unknown-address + 4, addrspace 1)
; SI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 6
; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
- ; SI-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD2]](p1) :: (store (s16) into unknown-address + 6, addrspace 1)
+ ; SI-NEXT: G_STORE [[LSHR1]](i32), [[PTR_ADD2]](p1) :: (store (s16) into unknown-address + 6, addrspace 1)
;
; CI-LABEL: name: test_store_global_v4s16_align2
; CI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
@@ -3586,21 +3586,21 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: G_STORE [[BITCAST]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: G_STORE [[BITCAST]](i32), [[COPY]](p1) :: (store (s16), addrspace 1)
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; VI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
+ ; VI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
; VI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
- ; VI-NEXT: G_STORE [[BITCAST1]](s32), [[PTR_ADD1]](p1) :: (store (s16) into unknown-address + 4, addrspace 1)
+ ; VI-NEXT: G_STORE [[BITCAST1]](i32), [[PTR_ADD1]](p1) :: (store (s16) into unknown-address + 4, addrspace 1)
; VI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 6
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
- ; VI-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD2]](p1) :: (store (s16) into unknown-address + 6, addrspace 1)
+ ; VI-NEXT: G_STORE [[LSHR1]](i32), [[PTR_ADD2]](p1) :: (store (s16) into unknown-address + 6, addrspace 1)
;
; GFX9-LABEL: name: test_store_global_v4s16_align2
; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sub.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sub.mir
index 9eee44a9ed9bf..77054314125cf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sub.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sub.mir
@@ -144,17 +144,18 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[BITCAST]], [[BITCAST1]]
; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[LSHR]], [[LSHR1]]
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -164,20 +165,21 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[TRUNC2]]
; GFX8-NEXT: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[TRUNC3]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -259,13 +261,13 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC5]](s16), [[DEF]](s16)
; GFX9-NEXT: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
; GFX9-NEXT: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[SUB]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[SUB1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[SUB]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[SUB1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: S_ENDPGM 0, implicit [[TRUNC6]](s16), implicit [[TRUNC7]](s16), implicit [[TRUNC8]](s16)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
@@ -298,16 +300,16 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[BITCAST]], [[BITCAST2]]
; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[LSHR]], [[LSHR2]]
; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[BITCAST1]], [[BITCAST3]]
@@ -315,12 +317,13 @@ body: |
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB2]], [[C1]]
; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB3]], [[C1]]
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -332,36 +335,37 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX8-NEXT: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[TRUNC4]]
; GFX8-NEXT: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[TRUNC5]]
; GFX8-NEXT: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[TRUNC2]], [[TRUNC6]]
; GFX8-NEXT: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[TRUNC3]], [[TRUNC7]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SUB3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir
index a2b60a4fcb035..e080c86e08555 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir
@@ -128,11 +128,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
@@ -144,14 +144,15 @@ body: |
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[ADD1]](s32), [[AND3]]
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND2]](s32)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AND3]](s32)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY2]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C2]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[AND5]](s32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
; CHECK-NEXT: $vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<2 x s32>)
@@ -174,14 +175,14 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
@@ -201,21 +202,22 @@ body: |
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP2]](s1)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND2]](s32)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AND3]](s32)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY2]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[AND6]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[COPY4]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL1]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
- ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
- ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
+ ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
+ ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND7]](s32), [[AND8]](s32), [[AND9]](s32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
@@ -243,16 +245,16 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr1_vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
@@ -272,12 +274,13 @@ body: |
; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[ADD3]](s32), [[AND7]]
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND2]](s32)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AND3]](s32)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY2]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[AND6]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[AND7]](s32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C2]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[COPY4]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -285,11 +288,11 @@ body: |
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP2]](s1)
; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP3]](s1)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C2]]
- ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C2]]
- ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C2]]
- ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
+ ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
+ ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C3]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND8]](s32), [[AND9]](s32), [[AND10]](s32), [[AND11]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
; CHECK-NEXT: $vgpr2_vgpr3_vgpr4_vgpr5 = COPY [[BUILD_VECTOR]](<4 x s32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
index e77005faf90ed..b6314482721b4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
@@ -192,7 +192,6 @@ body: |
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
- ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
@@ -200,20 +199,21 @@ body: |
; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
- ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C2]](s16), [[C2]](s16)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[BUILD_VECTOR]], [[BUILD_VECTOR2]](<2 x s16>)
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[BUILD_VECTOR1]], [[BUILD_VECTOR2]](<2 x s16>)
; GFX9-NEXT: [[UADDSAT:%[0-9]+]]:_(<2 x s16>) = G_UADDSAT [[SHL]], [[SHL1]]
; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UADDSAT]], [[BUILD_VECTOR2]](<2 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LSHR2]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LSHR2]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C2]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
; GFX9-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C3]]
; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C3]]
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C2]](s16)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
; GFX9-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL2]]
; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
@@ -291,25 +291,26 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
- ; GFX6-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C1]]
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+ ; GFX6-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C2]]
; GFX6-NEXT: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[SHL1]]
; GFX6-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[UMIN]]
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
- ; GFX6-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[SHL2]], [[C1]]
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX6-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[SHL2]], [[C2]]
; GFX6-NEXT: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[SHL3]]
; GFX6-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[UMIN1]]
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C]](s32)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR2]], [[SHL4]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -319,20 +320,21 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC2]]
; GFX8-NEXT: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC1]], [[TRUNC3]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -361,40 +363,41 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
- ; GFX6-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C1]]
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+ ; GFX6-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C2]]
; GFX6-NEXT: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[SHL1]]
; GFX6-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[UMIN]]
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[SHL2]], [[C1]]
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C1]](s32)
+ ; GFX6-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[SHL2]], [[C2]]
; GFX6-NEXT: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[SHL3]]
; GFX6-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[UMIN1]]
- ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C]](s32)
- ; GFX6-NEXT: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SHL4]], [[C1]]
+ ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C1]](s32)
+ ; GFX6-NEXT: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SHL4]], [[C2]]
; GFX6-NEXT: [[UMIN2:%[0-9]+]]:_(s32) = G_UMIN [[XOR2]], [[SHL5]]
; GFX6-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SHL4]], [[UMIN2]]
- ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR4]], [[C]](s32)
+ ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR4]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR3]], [[SHL6]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR5]], [[SHL7]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL7]]
+ ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL7]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -404,33 +407,34 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX8-NEXT: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC3]]
; GFX8-NEXT: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC1]], [[TRUNC4]]
; GFX8-NEXT: [[UADDSAT2:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC2]], [[TRUNC5]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT2]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -440,25 +444,25 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
; GFX9-NEXT: [[UADDSAT:%[0-9]+]]:_(<2 x s16>) = G_UADDSAT [[UV]], [[BUILD_VECTOR]]
; GFX9-NEXT: [[UADDSAT1:%[0-9]+]]:_(<2 x s16>) = G_UADDSAT [[UV1]], [[BUILD_VECTOR1]]
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UADDSAT]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UADDSAT1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UADDSAT]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UADDSAT1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC3]](s16), [[TRUNC4]](s16)
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC5]](s16), [[DEF]](s16)
; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
@@ -484,45 +488,46 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
- ; GFX6-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C1]]
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C1]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+ ; GFX6-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C2]]
; GFX6-NEXT: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[SHL1]]
; GFX6-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[UMIN]]
- ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C]](s32)
- ; GFX6-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[SHL2]], [[C1]]
+ ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C1]](s32)
+ ; GFX6-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[SHL2]], [[C2]]
; GFX6-NEXT: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[SHL3]]
; GFX6-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[UMIN1]]
- ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[BITCAST3]], [[C]](s32)
- ; GFX6-NEXT: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SHL4]], [[C1]]
+ ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[BITCAST3]], [[C1]](s32)
+ ; GFX6-NEXT: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SHL4]], [[C2]]
; GFX6-NEXT: [[UMIN2:%[0-9]+]]:_(s32) = G_UMIN [[XOR2]], [[SHL5]]
; GFX6-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SHL4]], [[UMIN2]]
- ; GFX6-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C]](s32)
- ; GFX6-NEXT: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SHL6]], [[C1]]
+ ; GFX6-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C1]](s32)
+ ; GFX6-NEXT: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SHL6]], [[C2]]
; GFX6-NEXT: [[UMIN3:%[0-9]+]]:_(s32) = G_UMIN [[XOR3]], [[SHL7]]
; GFX6-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SHL6]], [[UMIN3]]
- ; GFX6-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD3]], [[C]](s32)
- ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR5]], [[C]](s32)
+ ; GFX6-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD3]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR5]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR4]], [[SHL8]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[LSHR7]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[LSHR7]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR6]], [[SHL9]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -534,36 +539,37 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX8-NEXT: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC4]]
; GFX8-NEXT: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC1]], [[TRUNC5]]
; GFX8-NEXT: [[UADDSAT2:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC2]], [[TRUNC6]]
; GFX8-NEXT: [[UADDSAT3:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC3]], [[TRUNC7]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT2]](s16)
; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir
index 54da5f43ba7ac..76563fd45cf1b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir
@@ -2294,11 +2294,11 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
@@ -2324,7 +2324,7 @@ body: |
; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](i32), [[AND1]]
; GFX6-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[SELECT]], [[C4]]
; GFX6-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
- ; GFX6-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](s32)
+ ; GFX6-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](i32)
; GFX6-NEXT: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(f32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](f32)
; GFX6-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C2]]
; GFX6-NEXT: [[FPTOUI1:%[0-9]+]]:_(i32) = G_FPTOUI [[FMUL1]](f32)
@@ -2345,7 +2345,8 @@ body: |
; GFX6-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD5]], [[SELECT3]]
; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT2]], [[C1]]
; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SELECT5]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -2355,11 +2356,11 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
@@ -2385,7 +2386,7 @@ body: |
; GFX8-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT1]](i32), [[AND1]]
; GFX8-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[SELECT]], [[C4]]
; GFX8-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
- ; GFX8-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](s32)
+ ; GFX8-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](i32)
; GFX8-NEXT: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(f32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](f32)
; GFX8-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C2]]
; GFX8-NEXT: [[FPTOUI1:%[0-9]+]]:_(i32) = G_FPTOUI [[FMUL1]](f32)
@@ -2406,7 +2407,8 @@ body: |
; GFX8-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD5]], [[SELECT3]]
; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT2]], [[C1]]
; GFX8-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SELECT5]], [[C1]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -2416,11 +2418,11 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX9-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
@@ -2447,7 +2449,7 @@ body: |
; GFX9-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[SELECT]], [[C4]]
; GFX9-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SELECT2]](s32)
- ; GFX9-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](s32)
+ ; GFX9-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](i32)
; GFX9-NEXT: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(f32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](f32)
; GFX9-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C2]]
; GFX9-NEXT: [[FPTOUI1:%[0-9]+]]:_(i32) = G_FPTOUI [[FMUL1]](f32)
@@ -2475,11 +2477,11 @@ body: |
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX10-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; GFX10-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
@@ -2506,7 +2508,7 @@ body: |
; GFX10-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[SELECT]], [[C4]]
; GFX10-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[ADD2]], [[SELECT]]
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SELECT2]](s32)
- ; GFX10-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](s32)
+ ; GFX10-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](i32)
; GFX10-NEXT: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(f32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](f32)
; GFX10-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C2]]
; GFX10-NEXT: [[FPTOUI1:%[0-9]+]]:_(i32) = G_FPTOUI [[FMUL1]](f32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir
index d5a8c28dcf562..83522167ee691 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir
@@ -411,17 +411,18 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[UMAX:%[0-9]+]]:_(s32) = G_UMAX [[AND]], [[AND1]]
; SI-NEXT: [[UMAX1:%[0-9]+]]:_(s32) = G_UMAX [[LSHR]], [[LSHR1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UMAX1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UMAX1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[UMAX]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -431,20 +432,21 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UMAX:%[0-9]+]]:_(s16) = G_UMAX [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[UMAX1:%[0-9]+]]:_(s16) = G_UMAX [[TRUNC1]], [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UMAX]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UMAX1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -507,11 +509,11 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX9-NEXT: [[UMAX:%[0-9]+]]:_(<2 x s16>) = G_UMAX [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
; GFX9-NEXT: [[UMAX1:%[0-9]+]]:_(s16) = G_UMAX [[DEF]], [[DEF]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UMAX]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UMAX]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UMAX1]](s16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[ANYEXT]](s32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[ANYEXT]](s32)
; GFX9-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x s32>)
;
; GFX1250-LABEL: name: test_umax_v3s16
@@ -522,11 +524,11 @@ body: |
; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX1250-NEXT: [[UMAX:%[0-9]+]]:_(<2 x s16>) = G_UMAX [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
; GFX1250-NEXT: [[UMAX1:%[0-9]+]]:_(s16) = G_UMAX [[DEF]], [[DEF]]
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UMAX]](<2 x s16>)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UMAX]](<2 x s16>)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UMAX1]](s16)
- ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[ANYEXT]](s32)
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[ANYEXT]](s32)
; GFX1250-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x s32>)
%0:_(<3 x s16>) = G_IMPLICIT_DEF
%1:_(<3 x s16>) = G_IMPLICIT_DEF
@@ -547,16 +549,16 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
@@ -566,10 +568,11 @@ body: |
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[UMAX2:%[0-9]+]]:_(s32) = G_UMAX [[AND2]], [[AND3]]
; SI-NEXT: [[UMAX3:%[0-9]+]]:_(s32) = G_UMAX [[LSHR1]], [[LSHR3]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UMAX1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UMAX1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[UMAX]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UMAX3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UMAX3]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[UMAX2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -581,36 +584,37 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[UMAX:%[0-9]+]]:_(s16) = G_UMAX [[TRUNC]], [[TRUNC4]]
; VI-NEXT: [[UMAX1:%[0-9]+]]:_(s16) = G_UMAX [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[UMAX2:%[0-9]+]]:_(s16) = G_UMAX [[TRUNC2]], [[TRUNC6]]
; VI-NEXT: [[UMAX3:%[0-9]+]]:_(s16) = G_UMAX [[TRUNC3]], [[TRUNC7]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UMAX]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UMAX1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UMAX2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UMAX3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir
index 06778bfcc8589..99c6ad23e22bc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir
@@ -411,17 +411,18 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[AND]], [[AND1]]
; SI-NEXT: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[LSHR]], [[LSHR1]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UMIN1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UMIN1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[UMIN]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -431,20 +432,21 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC1]], [[TRUNC3]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UMIN]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UMIN1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -507,11 +509,11 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX9-NEXT: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
; GFX9-NEXT: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[DEF]], [[DEF]]
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UMIN]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UMIN]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UMIN1]](s16)
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[ANYEXT]](s32)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[ANYEXT]](s32)
; GFX9-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x s32>)
;
; GFX1250-LABEL: name: test_umin_v3s16
@@ -522,11 +524,11 @@ body: |
; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
; GFX1250-NEXT: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
; GFX1250-NEXT: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[DEF]], [[DEF]]
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UMIN]](<2 x s16>)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UMIN]](<2 x s16>)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UMIN1]](s16)
- ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[ANYEXT]](s32)
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[ANYEXT]](s32)
; GFX1250-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x s32>)
%0:_(<3 x s16>) = G_IMPLICIT_DEF
%1:_(<3 x s16>) = G_IMPLICIT_DEF
@@ -547,16 +549,16 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
@@ -566,10 +568,11 @@ body: |
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[UMIN2:%[0-9]+]]:_(s32) = G_UMIN [[AND2]], [[AND3]]
; SI-NEXT: [[UMIN3:%[0-9]+]]:_(s32) = G_UMIN [[LSHR1]], [[LSHR3]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UMIN1]], [[C]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UMIN1]], [[C2]](s32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[UMIN]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UMIN3]], [[C]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UMIN3]], [[C2]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[UMIN2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -581,36 +584,37 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; VI-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC4]]
; VI-NEXT: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[UMIN2:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC2]], [[TRUNC6]]
; VI-NEXT: [[UMIN3:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC3]], [[TRUNC7]]
; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UMIN]](s16)
; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UMIN1]](s16)
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UMIN2]](s16)
; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UMIN3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulh.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulh.mir
index 801f29b2435ef..e896c69430386 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulh.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulh.mir
@@ -379,10 +379,11 @@ body: |
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C2]](i32)
; GFX8-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C]]
- ; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[LSHR2]](s32)
+ ; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[LSHR2]](i32)
; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
;
; GFX9-LABEL: name: test_umulh_v2s16
@@ -491,13 +492,13 @@ body: |
; GFX9-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C]]
; GFX9-NEXT: [[MUL1:%[0-9]+]]:_(s16) = G_MUL [[AND4]], [[AND5]]
; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[MUL1]], [[C1]](s16)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LSHR]](<2 x s16>)
- ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LSHR]](<2 x s16>)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C2]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
; GFX9-NEXT: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C]]
- ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C]]
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C1]](s16)
; GFX9-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[SHL]]
@@ -507,7 +508,8 @@ body: |
; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[LSHR1]], [[SHL1]]
; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
%0:_(s32) = COPY $vgpr0
@@ -584,11 +586,11 @@ body: |
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16)
; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[MUL]], [[BUILD_VECTOR2]](<2 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LSHR]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LSHR]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C2]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C]]
; GFX9-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C]]
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C1]](s16)
@@ -716,19 +718,20 @@ body: |
; GFX9-NEXT: [[MUL1:%[0-9]+]]:_(<2 x s16>) = G_MUL [[BUILD_VECTOR3]], [[BUILD_VECTOR4]]
; GFX9-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C4]](s16), [[C4]](s16)
; GFX9-NEXT: [[LSHR7:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[MUL1]], [[BUILD_VECTOR5]](<2 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LSHR6]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[LSHR7]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; GFX9-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; GFX9-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C5]]
- ; GFX9-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[LSHR8]], [[C5]]
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LSHR6]](<2 x s16>)
+ ; GFX9-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR8:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C5]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[LSHR7]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR9:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C5]](i32)
+ ; GFX9-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+ ; GFX9-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C6]]
+ ; GFX9-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[LSHR8]], [[C6]]
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C]](s32)
; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND8]], [[SHL]]
- ; GFX9-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C5]]
+ ; GFX9-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C6]]
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND10]], [[C1]](s32)
; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; GFX9-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[LSHR9]], [[C5]]
+ ; GFX9-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[LSHR9]], [[C6]]
; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C2]](s32)
; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulo.mir
index 9cf56d7d56e1f..d0f8e77bba035 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulo.mir
@@ -580,10 +580,11 @@ body: |
; GFX8-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C2]]
; GFX8-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C2]]
; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND8]](s32), [[AND9]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C3]](i32)
; GFX8-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C]]
- ; GFX8-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND10]](s32), [[LSHR]](s32)
+ ; GFX8-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND10]](s32), [[LSHR]](i32)
; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR1]](<2 x s32>)
; GFX8-NEXT: $vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s32>)
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values.mir
index fb9c18b965209..e45570422fc6c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values.mir
@@ -83,11 +83,11 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](s32)
- ; CHECK-NEXT: $vgpr1 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](i32)
+ ; CHECK-NEXT: $vgpr1 = COPY [[LSHR]](i32)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(s16), %2:_(s16) = G_UNMERGE_VALUES %0
%3:_(s32) = G_ANYEXT %1
@@ -126,15 +126,15 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](s32)
- ; CHECK-NEXT: $vgpr1 = COPY [[LSHR]](s32)
- ; CHECK-NEXT: $vgpr2 = COPY [[BITCAST1]](s32)
- ; CHECK-NEXT: $vgpr3 = COPY [[LSHR1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](i32)
+ ; CHECK-NEXT: $vgpr1 = COPY [[LSHR]](i32)
+ ; CHECK-NEXT: $vgpr2 = COPY [[BITCAST1]](i32)
+ ; CHECK-NEXT: $vgpr3 = COPY [[LSHR1]](i32)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
%1:_(s16), %2:_(s16), %3:_(s16), %4:_(s16) = G_UNMERGE_VALUES %0
%5:_(s32) = G_ANYEXT %1
@@ -157,19 +157,19 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](s32)
- ; CHECK-NEXT: $vgpr1 = COPY [[LSHR]](s32)
- ; CHECK-NEXT: $vgpr2 = COPY [[BITCAST1]](s32)
- ; CHECK-NEXT: $vgpr3 = COPY [[LSHR1]](s32)
- ; CHECK-NEXT: $vgpr4 = COPY [[BITCAST2]](s32)
- ; CHECK-NEXT: $vgpr5 = COPY [[LSHR2]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](i32)
+ ; CHECK-NEXT: $vgpr1 = COPY [[LSHR]](i32)
+ ; CHECK-NEXT: $vgpr2 = COPY [[BITCAST1]](i32)
+ ; CHECK-NEXT: $vgpr3 = COPY [[LSHR1]](i32)
+ ; CHECK-NEXT: $vgpr4 = COPY [[BITCAST2]](i32)
+ ; CHECK-NEXT: $vgpr5 = COPY [[LSHR2]](i32)
%0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
%1:_(s16), %2:_(s16), %3:_(s16), %4:_(s16), %5:_(s16), %6:_(s16) = G_UNMERGE_VALUES %0
%7:_(s32) = G_ANYEXT %1
@@ -587,11 +587,11 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p3)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[PTRTOINT]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[PTRTOINT]](s32)
- ; CHECK-NEXT: $vgpr1 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(i32) = G_PTRTOINT [[COPY]](p3)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[PTRTOINT]], [[C]](i32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[PTRTOINT]](i32)
+ ; CHECK-NEXT: $vgpr1 = COPY [[LSHR]](i32)
%0:_(p3) = COPY $vgpr0
%1:_(s16), %2:_(s16) = G_UNMERGE_VALUES %0
%3:_(s32) = G_ANYEXT %1
@@ -609,17 +609,17 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p3)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[PTRTOINT]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[PTRTOINT]], [[C1]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[PTRTOINT]], [[C2]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[PTRTOINT]](s32)
- ; CHECK-NEXT: $vgpr1 = COPY [[LSHR]](s32)
- ; CHECK-NEXT: $vgpr2 = COPY [[LSHR1]](s32)
- ; CHECK-NEXT: $vgpr3 = COPY [[LSHR2]](s32)
+ ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(i32) = G_PTRTOINT [[COPY]](p3)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[PTRTOINT]], [[C]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[PTRTOINT]], [[C1]](i32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[PTRTOINT]], [[C2]](i32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[PTRTOINT]](i32)
+ ; CHECK-NEXT: $vgpr1 = COPY [[LSHR]](i32)
+ ; CHECK-NEXT: $vgpr2 = COPY [[LSHR1]](i32)
+ ; CHECK-NEXT: $vgpr3 = COPY [[LSHR2]](i32)
%0:_(p3) = COPY $vgpr0
%1:_(s8), %2:_(s8), %3:_(s8), %4:_(s8) = G_UNMERGE_VALUES %0
%5:_(s32) = G_ANYEXT %1
@@ -1046,28 +1046,28 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<12 x s16>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<12 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[BITCAST1]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[BITCAST1]](i32)
; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>), [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<12 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR1]](s32), [[BITCAST3]](s32), [[LSHR2]](s32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV8]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR1]](i32), [[BITCAST3]](i32), [[LSHR2]](i32)
; CHECK-NEXT: [[UV12:%[0-9]+]]:_(<2 x s16>), [[UV13:%[0-9]+]]:_(<2 x s16>), [[UV14:%[0-9]+]]:_(<2 x s16>), [[UV15:%[0-9]+]]:_(<2 x s16>), [[UV16:%[0-9]+]]:_(<2 x s16>), [[UV17:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<12 x s16>)
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV15]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV16]](<2 x s16>)
- ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST4]](s32), [[LSHR3]](s32), [[BITCAST5]](s32)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV15]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV16]](<2 x s16>)
+ ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST4]](i32), [[LSHR3]](i32), [[BITCAST5]](i32)
; CHECK-NEXT: [[UV18:%[0-9]+]]:_(<2 x s16>), [[UV19:%[0-9]+]]:_(<2 x s16>), [[UV20:%[0-9]+]]:_(<2 x s16>), [[UV21:%[0-9]+]]:_(<2 x s16>), [[UV22:%[0-9]+]]:_(<2 x s16>), [[UV23:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<12 x s16>)
- ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV22]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[UV23]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST7]], [[C]](s32)
- ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR4]](s32), [[BITCAST7]](s32), [[LSHR5]](s32)
+ ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[UV22]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[UV23]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR4]](i32), [[BITCAST7]](i32), [[LSHR5]](i32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
; CHECK-NEXT: $vgpr3_vgpr4_vgpr5 = COPY [[BUILD_VECTOR1]](<3 x s32>)
; CHECK-NEXT: $vgpr6_vgpr7_vgpr8 = COPY [[BUILD_VECTOR2]](<3 x s32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir
index dcba1cc568fa3..f42193423291b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir
@@ -2206,11 +2206,11 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
@@ -2233,7 +2233,7 @@ body: |
; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](i32), [[AND1]]
; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(i32) = G_SUB [[SELECT]], [[AND1]]
; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
- ; GFX6-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](s32)
+ ; GFX6-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](i32)
; GFX6-NEXT: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(f32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](f32)
; GFX6-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C2]]
; GFX6-NEXT: [[FPTOUI1:%[0-9]+]]:_(i32) = G_FPTOUI [[FMUL1]](f32)
@@ -2252,7 +2252,8 @@ body: |
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB7]], [[SELECT2]]
; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT1]], [[C1]]
; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SELECT3]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -2262,11 +2263,11 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
@@ -2289,7 +2290,7 @@ body: |
; GFX8-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](i32), [[AND1]]
; GFX8-NEXT: [[SUB3:%[0-9]+]]:_(i32) = G_SUB [[SELECT]], [[AND1]]
; GFX8-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
- ; GFX8-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](s32)
+ ; GFX8-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](i32)
; GFX8-NEXT: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(f32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](f32)
; GFX8-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C2]]
; GFX8-NEXT: [[FPTOUI1:%[0-9]+]]:_(i32) = G_FPTOUI [[FMUL1]](f32)
@@ -2308,7 +2309,8 @@ body: |
; GFX8-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB7]], [[SELECT2]]
; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT1]], [[C1]]
; GFX8-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SELECT3]], [[C1]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -2318,11 +2320,11 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX9-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
@@ -2346,7 +2348,7 @@ body: |
; GFX9-NEXT: [[SUB3:%[0-9]+]]:_(i32) = G_SUB [[SELECT]], [[AND1]]
; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SELECT1]](s32)
- ; GFX9-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](s32)
+ ; GFX9-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](i32)
; GFX9-NEXT: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(f32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](f32)
; GFX9-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C2]]
; GFX9-NEXT: [[FPTOUI1:%[0-9]+]]:_(i32) = G_FPTOUI [[FMUL1]](f32)
@@ -2372,11 +2374,11 @@ body: |
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX10-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; GFX10-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
@@ -2400,7 +2402,7 @@ body: |
; GFX10-NEXT: [[SUB3:%[0-9]+]]:_(i32) = G_SUB [[SELECT]], [[AND1]]
; GFX10-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SELECT1]](s32)
- ; GFX10-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](s32)
+ ; GFX10-NEXT: [[UITOFP1:%[0-9]+]]:_(f32) = G_UITOFP [[LSHR1]](i32)
; GFX10-NEXT: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(f32) = G_AMDGPU_RCP_IFLAG [[UITOFP1]](f32)
; GFX10-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C2]]
; GFX10-NEXT: [[FPTOUI1:%[0-9]+]]:_(i32) = G_FPTOUI [[FMUL1]](f32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ushlsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ushlsat.mir
index 956ab79d5447c..b3031992fefff 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ushlsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ushlsat.mir
@@ -337,27 +337,28 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C2]](s32)
; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[AND]](s32)
; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[SHL1]], [[AND]](s32)
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL]](s32), [[LSHR2]]
- ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C2]], [[SHL1]]
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[SELECT]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[SHL2]], [[LSHR1]](s32)
- ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[SHL3]], [[LSHR1]](s32)
+ ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C3]], [[SHL1]]
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[SELECT]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[SHL2]], [[LSHR1]](i32)
+ ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[SHL3]], [[LSHR1]](i32)
; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL2]](s32), [[LSHR4]]
- ; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[C2]], [[SHL3]]
- ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SELECT1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LSHR5]], [[C]](s32)
+ ; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[C3]], [[SHL3]]
+ ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SELECT1]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LSHR5]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR3]], [[SHL4]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -367,15 +368,15 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC2]](s16)
; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[SHL]], [[TRUNC2]](s16)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -387,7 +388,8 @@ body: |
; GFX8-NEXT: [[SELECT1:%[0-9]+]]:_(s16) = G_SELECT [[ICMP1]](s1), [[C1]], [[SHL1]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -397,15 +399,15 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC2]](s16)
; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[SHL]], [[TRUNC2]](s16)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -434,42 +436,43 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[LSHR1]](s32)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[SHL1]], [[LSHR1]](s32)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[LSHR1]](i32)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[SHL1]], [[LSHR1]](i32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL]](s32), [[LSHR3]]
- ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C1]], [[SHL1]]
- ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[SELECT]], [[C]](s32)
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C2]]
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C2]], [[SHL1]]
+ ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[SELECT]], [[C1]](s32)
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C3]]
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[SHL2]], [[AND]](s32)
; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SHL3]], [[AND]](s32)
; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL2]](s32), [[LSHR5]]
- ; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[C1]], [[SHL3]]
- ; GFX6-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SELECT1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[SHL4]], [[LSHR2]](s32)
- ; GFX6-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[SHL5]], [[LSHR2]](s32)
+ ; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[C2]], [[SHL3]]
+ ; GFX6-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SELECT1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[SHL4]], [[LSHR2]](i32)
+ ; GFX6-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[SHL5]], [[LSHR2]](i32)
; GFX6-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL4]](s32), [[LSHR7]]
- ; GFX6-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[C1]], [[SHL5]]
- ; GFX6-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[SELECT2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR6]], [[C]](s32)
+ ; GFX6-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[C2]], [[SHL5]]
+ ; GFX6-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[SELECT2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR6]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR4]], [[SHL6]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C]](s32)
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR8]], [[SHL7]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL7]]
+ ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL7]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -479,19 +482,19 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC3]](s16)
; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[SHL]], [[TRUNC3]](s16)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -507,15 +510,16 @@ body: |
; GFX8-NEXT: [[SELECT2:%[0-9]+]]:_(s16) = G_SELECT [[ICMP2]](s1), [[C1]], [[SHL2]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
- ; GFX8-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT2]](s16)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL4]]
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL4]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -525,19 +529,19 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC3]](s16)
; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[SHL]], [[TRUNC3]](s16)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -577,48 +581,49 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C2]](s32)
; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[AND]](s32)
; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[SHL1]], [[AND]](s32)
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
; GFX6-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL]](s32), [[LSHR4]]
- ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C2]], [[SHL1]]
- ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SELECT]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[SHL2]], [[LSHR2]](s32)
- ; GFX6-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SHL3]], [[LSHR2]](s32)
+ ; GFX6-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C3]], [[SHL1]]
+ ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SELECT]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[SHL2]], [[LSHR2]](i32)
+ ; GFX6-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SHL3]], [[LSHR2]](i32)
; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL2]](s32), [[LSHR6]]
- ; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[C2]], [[SHL3]]
- ; GFX6-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[SELECT1]], [[C]](s32)
+ ; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[C3]], [[SHL3]]
+ ; GFX6-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[SELECT1]], [[C2]](s32)
; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C2]](s32)
; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[SHL4]], [[AND1]](s32)
; GFX6-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[SHL5]], [[AND1]](s32)
; GFX6-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL4]](s32), [[LSHR8]]
- ; GFX6-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[C2]], [[SHL5]]
- ; GFX6-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[SELECT2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[SHL6]], [[LSHR3]](s32)
- ; GFX6-NEXT: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[SHL7]], [[LSHR3]](s32)
+ ; GFX6-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[C3]], [[SHL5]]
+ ; GFX6-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[SELECT2]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[SHL6]], [[LSHR3]](i32)
+ ; GFX6-NEXT: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[SHL7]], [[LSHR3]](i32)
; GFX6-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL6]](s32), [[LSHR10]]
- ; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[C2]], [[SHL7]]
- ; GFX6-NEXT: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[SELECT3]], [[C]](s32)
- ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR7]], [[C]](s32)
+ ; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[C3]], [[SHL7]]
+ ; GFX6-NEXT: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[SELECT3]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR7]], [[C2]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR5]], [[SHL8]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[LSHR11]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[LSHR11]], [[C2]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR9]], [[SHL9]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -630,24 +635,24 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC4]](s16)
; GFX8-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[SHL]], [[TRUNC4]](s16)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
@@ -667,12 +672,13 @@ body: |
; GFX8-NEXT: [[SELECT3:%[0-9]+]]:_(s16) = G_SELECT [[ICMP3]](s1), [[C1]], [[SHL3]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
- ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT2]](s16)
; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT3]](s16)
- ; GFX8-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -684,24 +690,24 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC4]](s16)
; GFX9-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[SHL]], [[TRUNC4]](s16)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir
index ce0b17fe11f48..114b794044dd5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir
@@ -128,11 +128,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
@@ -144,14 +144,15 @@ body: |
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SUB1]](s32), [[AND3]]
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND2]](s32)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AND3]](s32)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY2]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C2]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[AND5]](s32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
; CHECK-NEXT: $vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<2 x s32>)
@@ -174,14 +175,14 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr3_vgpr4_vgpr5
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<6 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
@@ -201,21 +202,22 @@ body: |
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP2]](s1)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND2]](s32)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AND3]](s32)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY2]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[AND6]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[COPY4]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL1]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
- ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
- ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
+ ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
+ ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND7]](s32), [[AND8]](s32), [[AND9]](s32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
@@ -243,16 +245,16 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr1_vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
@@ -272,12 +274,13 @@ body: |
; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SUB3]](s32), [[AND7]]
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND2]](s32)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AND3]](s32)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY2]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[AND6]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[AND7]](s32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C2]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[COPY4]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -285,11 +288,11 @@ body: |
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP2]](s1)
; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP3]](s1)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C2]]
- ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C2]]
- ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C2]]
- ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
+ ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
+ ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C3]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND8]](s32), [[AND9]](s32), [[AND10]](s32), [[AND11]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
; CHECK-NEXT: $vgpr2_vgpr3_vgpr4_vgpr5 = COPY [[BUILD_VECTOR]](<4 x s32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
index 02a6e9553de04..a06862c927fac 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
@@ -185,7 +185,6 @@ body: |
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
- ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
@@ -193,20 +192,21 @@ body: |
; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
- ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
- ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C2]](s16), [[C2]](s16)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[BUILD_VECTOR]], [[BUILD_VECTOR2]](<2 x s16>)
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[BUILD_VECTOR1]], [[BUILD_VECTOR2]](<2 x s16>)
; GFX9-NEXT: [[USUBSAT:%[0-9]+]]:_(<2 x s16>) = G_USUBSAT [[SHL]], [[SHL1]]
; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[USUBSAT]], [[BUILD_VECTOR2]](<2 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LSHR2]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LSHR2]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C2]](i32)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
; GFX9-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C3]]
; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C3]]
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C2]](s16)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
; GFX9-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL2]]
; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
@@ -282,22 +282,23 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
; GFX6-NEXT: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[SHL]], [[SHL1]]
; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[UMIN]]
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
; GFX6-NEXT: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[SHL2]], [[SHL3]]
; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[UMIN1]]
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C]](s32)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR2]], [[SHL4]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -307,20 +308,21 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC2]]
; GFX8-NEXT: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC1]], [[TRUNC3]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -349,36 +351,37 @@ body: |
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
; GFX6-NEXT: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[SHL]], [[SHL1]]
; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[UMIN]]
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C]](s32)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C1]](s32)
; GFX6-NEXT: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[SHL2]], [[SHL3]]
; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[UMIN1]]
- ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C]](s32)
+ ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C1]](s32)
; GFX6-NEXT: [[UMIN2:%[0-9]+]]:_(s32) = G_UMIN [[SHL4]], [[SHL5]]
; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL4]], [[UMIN2]]
- ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR4]], [[C]](s32)
+ ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR4]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR3]], [[SHL6]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR5]], [[SHL7]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL7]]
+ ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL7]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -388,33 +391,34 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX8-NEXT: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC3]]
; GFX8-NEXT: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC1]], [[TRUNC4]]
; GFX8-NEXT: [[USUBSAT2:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC2]], [[TRUNC5]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT2]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -424,25 +428,25 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
; GFX9-NEXT: [[USUBSAT:%[0-9]+]]:_(<2 x s16>) = G_USUBSAT [[UV]], [[BUILD_VECTOR]]
; GFX9-NEXT: [[USUBSAT1:%[0-9]+]]:_(<2 x s16>) = G_USUBSAT [[UV1]], [[BUILD_VECTOR1]]
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[USUBSAT]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[USUBSAT1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[USUBSAT]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[USUBSAT1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC3]](s16), [[TRUNC4]](s16)
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC5]](s16), [[DEF]](s16)
; GFX9-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16)
@@ -468,40 +472,41 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX6-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C]](s32)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C]](s32)
+ ; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[BITCAST2]], [[C1]](s32)
; GFX6-NEXT: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[SHL]], [[SHL1]]
; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[UMIN]]
- ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C]](s32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
- ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C]](s32)
+ ; GFX6-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C1]](s32)
; GFX6-NEXT: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[SHL2]], [[SHL3]]
; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[UMIN1]]
- ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[BITCAST3]], [[C]](s32)
+ ; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[BITCAST3]], [[C1]](s32)
; GFX6-NEXT: [[UMIN2:%[0-9]+]]:_(s32) = G_UMIN [[SHL4]], [[SHL5]]
; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL4]], [[UMIN2]]
- ; GFX6-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB2]], [[C]](s32)
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C]](s32)
+ ; GFX6-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB2]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C1]](s32)
; GFX6-NEXT: [[UMIN3:%[0-9]+]]:_(s32) = G_UMIN [[SHL6]], [[SHL7]]
; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SHL6]], [[UMIN3]]
- ; GFX6-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[SUB3]], [[C]](s32)
- ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR5]], [[C]](s32)
+ ; GFX6-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[SUB3]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR5]], [[C1]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR4]], [[SHL8]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[LSHR7]], [[C]](s32)
+ ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[LSHR7]], [[C1]](s32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR6]], [[SHL9]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -513,36 +518,37 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
; GFX8-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; GFX8-NEXT: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC4]]
; GFX8-NEXT: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC1]], [[TRUNC5]]
; GFX8-NEXT: [[USUBSAT2:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC2]], [[TRUNC6]]
; GFX8-NEXT: [[USUBSAT3:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC3]], [[TRUNC7]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT2]](s16)
; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx7.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx7.mir
index 4328d47969a1e..cc992622e6f12 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx7.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx7.mir
@@ -50,11 +50,11 @@ body: |
; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY3]](s32)
; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[BUILD_VECTOR1]](<2 x s32>)
; GFX7-NEXT: [[AND:%[0-9]+]]:_(<2 x s16>) = G_AND [[TRUNC]], [[TRUNC1]]
- ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[AND]](<2 x s16>)
- ; GFX7-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX7-NEXT: $vgpr0 = COPY [[BITCAST]](s32)
- ; GFX7-NEXT: $vgpr1 = COPY [[LSHR]](s32)
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[AND]](<2 x s16>)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX7-NEXT: $vgpr0 = COPY [[BITCAST]](i32)
+ ; GFX7-NEXT: $vgpr1 = COPY [[LSHR]](i32)
; GFX7-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
%3:_(s32) = COPY $vgpr0
%4:_(s32) = COPY $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx8-plus.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx8-plus.mir
index ad4f068ca5842..800feb0d7c978 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx8-plus.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx8-plus.mir
@@ -74,32 +74,33 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; GFX8-NEXT: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[TRUNC3]]
; GFX8-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[TRUNC4]]
; GFX8-NEXT: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[TRUNC5]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST4]](<2 x s16>)
@@ -149,32 +150,33 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC3]](s16)
; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC4]](s16)
; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[TRUNC5]](s16)
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SHL]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SHL1]](s16)
- ; GFX8-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SHL2]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST4]](<2 x s16>)
@@ -186,12 +188,12 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; GFX9-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY3]](<2 x s16>)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
; GFX9-NEXT: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[COPY]], [[COPY2]](<2 x s16>)
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
@@ -234,43 +236,44 @@ body: |
; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
; GFX8-NEXT: [[COPY4:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
; GFX8-NEXT: [[COPY5:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
- ; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY4]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; GFX8-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
- ; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[COPY5]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
- ; GFX8-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[COPY4]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; GFX8-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](i32)
+ ; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[COPY5]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
+ ; GFX8-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](i32)
; GFX8-NEXT: [[FMA:%[0-9]+]]:_(s16) = G_FMA [[TRUNC]], [[TRUNC4]], [[TRUNC8]]
; GFX8-NEXT: [[FMA1:%[0-9]+]]:_(s16) = G_FMA [[TRUNC1]], [[TRUNC5]], [[TRUNC9]]
; GFX8-NEXT: [[FMA2:%[0-9]+]]:_(s16) = G_FMA [[TRUNC2]], [[TRUNC6]], [[TRUNC10]]
; GFX8-NEXT: [[FMA3:%[0-9]+]]:_(s16) = G_FMA [[TRUNC3]], [[TRUNC7]], [[TRUNC11]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMA]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMA1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMA2]](s16)
; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMA3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
@@ -319,30 +322,30 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY2]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](i32)
; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
; GFX8-NEXT: [[COPY4:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
; GFX8-NEXT: [[COPY5:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
- ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
- ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY4]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
- ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
- ; GFX8-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
- ; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[COPY5]](<2 x s16>)
- ; GFX8-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
+ ; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY3]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[COPY4]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](i32)
+ ; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX8-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[COPY5]](<2 x s16>)
+ ; GFX8-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](i32)
; GFX8-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
; GFX8-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC5]]
; GFX8-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]]
@@ -360,17 +363,18 @@ body: |
; GFX8-NEXT: [[FMAXNUM_IEEE4:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE8]], [[FCANONICALIZE9]]
; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE2]](s16)
; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE4]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL2]]
; GFX8-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
index 9a93c7db6c920..2ee70e3578111 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
@@ -516,9 +516,10 @@ body: |
; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<4 x s16>) = G_XOR [[CONCAT_VECTORS]], [[CONCAT_VECTORS1]]
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[XOR]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C2]](i32)
+ ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL4]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
@@ -535,15 +536,15 @@ body: |
; CHECK-NEXT: [[CONCAT_VECTORS3:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST8]](<2 x s16>), [[BITCAST9]](<2 x s16>)
; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(<4 x s16>) = G_XOR [[CONCAT_VECTORS2]], [[CONCAT_VECTORS3]]
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[XOR1]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST10]], [[C1]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST4]], [[C2]]
+ ; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C2]](i32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST4]], [[C3]]
; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
; CHECK-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL8]]
; CHECK-NEXT: [[BITCAST11:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR8]](i32)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST5]], [[C2]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST10]], [[C2]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST5]], [[C3]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST10]], [[C3]]
; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL9]]
; CHECK-NEXT: [[BITCAST12:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR9]](i32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-zext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-zext.mir
index 46d22bdbad545..9d4fcad6b03eb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-zext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-zext.mir
@@ -133,12 +133,12 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](s32)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(<2 x s32>) = G_ZEXT %0
@@ -156,14 +156,14 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](s32), [[AND1]](s32)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](i32), [[AND1]](s32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
%1:_(<3 x s16>) = G_EXTRACT %0, 0
@@ -182,15 +182,15 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](s32), [[AND1]](s32), [[LSHR1]](s32)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](i32), [[AND1]](s32), [[LSHR1]](i32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
%1:_(<4 x s32>) = G_ZEXT %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll
index d1b327ca3a1e0..2ec6b01510975 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -o - %s | FileCheck -check-prefix=GFX81 %s
; FIXME: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX9 %s
; FIXME: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
@@ -39,17 +39,16 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
define amdgpu_ps void @image_store_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <2 x half> %in) {
; UNPACKED-LABEL: image_store_v2f16:
; UNPACKED: ; %bb.0:
-; UNPACKED-NEXT: s_mov_b32 s11, s9
-; UNPACKED-NEXT: s_mov_b32 s10, s8
-; UNPACKED-NEXT: s_mov_b32 s9, s7
-; UNPACKED-NEXT: s_mov_b32 s8, s6
-; UNPACKED-NEXT: s_mov_b32 s7, s5
-; UNPACKED-NEXT: s_mov_b32 s6, s4
-; UNPACKED-NEXT: s_mov_b32 s5, s3
-; UNPACKED-NEXT: s_mov_b32 s4, s2
+; UNPACKED-NEXT: s_mov_b32 s0, s2
+; UNPACKED-NEXT: s_mov_b32 s1, s3
+; UNPACKED-NEXT: s_mov_b32 s2, s4
+; UNPACKED-NEXT: s_mov_b32 s3, s5
+; UNPACKED-NEXT: s_mov_b32 s4, s6
+; UNPACKED-NEXT: s_mov_b32 s5, s7
+; UNPACKED-NEXT: s_mov_b32 s6, s8
+; UNPACKED-NEXT: s_mov_b32 s7, s9
; UNPACKED-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; UNPACKED-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; UNPACKED-NEXT: image_store v[2:3], v[0:1], s[4:11] dmask:0x3 unorm d16
+; UNPACKED-NEXT: image_store v[2:3], v[0:1], s[0:7] dmask:0x3 unorm d16
; UNPACKED-NEXT: s_endpgm
;
; GFX81-LABEL: image_store_v2f16:
@@ -71,18 +70,19 @@ define amdgpu_ps void @image_store_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <3 x half> %in) {
; UNPACKED-LABEL: image_store_v3f16:
; UNPACKED: ; %bb.0:
-; UNPACKED-NEXT: s_mov_b32 s11, s9
-; UNPACKED-NEXT: s_mov_b32 s10, s8
-; UNPACKED-NEXT: s_mov_b32 s9, s7
-; UNPACKED-NEXT: s_mov_b32 s8, s6
-; UNPACKED-NEXT: s_mov_b32 s7, s5
-; UNPACKED-NEXT: s_mov_b32 s6, s4
-; UNPACKED-NEXT: s_mov_b32 s5, s3
-; UNPACKED-NEXT: s_mov_b32 s4, s2
-; UNPACKED-NEXT: v_and_b32_e32 v4, 0xffff, v3
-; UNPACKED-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; UNPACKED-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; UNPACKED-NEXT: image_store v[2:4], v[0:1], s[4:11] dmask:0x7 unorm d16
+; UNPACKED-NEXT: v_mov_b32_e32 v5, v1
+; UNPACKED-NEXT: v_mov_b32_e32 v1, v2
+; UNPACKED-NEXT: s_mov_b32 s0, s2
+; UNPACKED-NEXT: s_mov_b32 s1, s3
+; UNPACKED-NEXT: s_mov_b32 s2, s4
+; UNPACKED-NEXT: s_mov_b32 s3, s5
+; UNPACKED-NEXT: s_mov_b32 s4, s6
+; UNPACKED-NEXT: s_mov_b32 s5, s7
+; UNPACKED-NEXT: s_mov_b32 s6, s8
+; UNPACKED-NEXT: s_mov_b32 s7, s9
+; UNPACKED-NEXT: v_mov_b32_e32 v4, v0
+; UNPACKED-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; UNPACKED-NEXT: image_store v[1:3], v[4:5], s[0:7] dmask:0x7 unorm d16
; UNPACKED-NEXT: s_endpgm
;
; GFX81-LABEL: image_store_v3f16:
@@ -104,19 +104,20 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
define amdgpu_ps void @image_store_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <4 x half> %in) {
; UNPACKED-LABEL: image_store_v4f16:
; UNPACKED: ; %bb.0:
-; UNPACKED-NEXT: s_mov_b32 s11, s9
-; UNPACKED-NEXT: s_mov_b32 s10, s8
-; UNPACKED-NEXT: s_mov_b32 s9, s7
-; UNPACKED-NEXT: s_mov_b32 s8, s6
-; UNPACKED-NEXT: s_mov_b32 s7, s5
-; UNPACKED-NEXT: s_mov_b32 s6, s4
-; UNPACKED-NEXT: s_mov_b32 s5, s3
-; UNPACKED-NEXT: s_mov_b32 s4, s2
-; UNPACKED-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; UNPACKED-NEXT: v_and_b32_e32 v4, 0xffff, v3
-; UNPACKED-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; UNPACKED-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; UNPACKED-NEXT: image_store v[2:5], v[0:1], s[4:11] dmask:0xf unorm d16
+; UNPACKED-NEXT: v_mov_b32_e32 v6, v1
+; UNPACKED-NEXT: v_mov_b32_e32 v1, v2
+; UNPACKED-NEXT: s_mov_b32 s0, s2
+; UNPACKED-NEXT: s_mov_b32 s1, s3
+; UNPACKED-NEXT: s_mov_b32 s2, s4
+; UNPACKED-NEXT: s_mov_b32 s3, s5
+; UNPACKED-NEXT: s_mov_b32 s4, s6
+; UNPACKED-NEXT: s_mov_b32 s5, s7
+; UNPACKED-NEXT: s_mov_b32 s6, s8
+; UNPACKED-NEXT: s_mov_b32 s7, s9
+; UNPACKED-NEXT: v_mov_b32_e32 v5, v0
+; UNPACKED-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; UNPACKED-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; UNPACKED-NEXT: image_store v[1:4], v[5:6], s[0:7] dmask:0xf unorm d16
; UNPACKED-NEXT: s_endpgm
;
; GFX81-LABEL: image_store_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
index 181fb3d122f9d..85871a6bc3be0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
@@ -107,20 +107,19 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr_128(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY7]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[COPY5]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16
@@ -161,24 +160,22 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr_128(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x s16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY8]](s32)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY5]](<2 x s16>)
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR1:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128(<4 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[BITCAST1]](s32), %subreg.sub2, [[LSHR1]](s32), %subreg.sub3
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE]](<4 x i32>), [[COPY6]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY7]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY5]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
@@ -225,47 +222,44 @@ define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr4
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr5
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr6
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x s16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY8]](s32)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY5]](<2 x s16>)
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR1:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128(<4 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[BITCAST1]](s32), %subreg.sub2, [[LSHR1]](s32), %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY5]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY]](i32), implicit $exec
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY1]](i32), implicit $exec
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY2]](i32), implicit $exec
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY3]](i32), implicit $exec
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[V_READFIRSTLANE_B32_]](i32), %subreg.sub0, [[V_READFIRSTLANE_B32_1]](i32), %subreg.sub1, [[V_READFIRSTLANE_B32_2]](i32), %subreg.sub2, [[V_READFIRSTLANE_B32_3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE1]].sub0_sub1(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE1]].sub2_sub3(<4 x i32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
- ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE]](<4 x i32>), [[COPY6]](i32), [[REG_SEQUENCE1]](<4 x i32>), [[COPY7]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
@@ -383,20 +377,19 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr_128(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sreg_32(i32) = G_CONSTANT i32 4095
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x f16>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY6]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[COPY5]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY6]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095
@@ -437,20 +430,19 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr_128(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sreg_32(i32) = G_CONSTANT i32 4096
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x f16>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY6]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[COPY5]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY6]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096
@@ -491,20 +483,19 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr_128(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY7]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[COPY5]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_16
@@ -546,20 +537,19 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr_128(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY7]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[COPY5]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4095
@@ -601,23 +591,22 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr_128(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[ADD:%[0-9]+]]:vgpr_32(i32) = G_ADD [[COPY5]], [[COPY7]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x f16>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY8]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[ADD]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4096
@@ -665,50 +654,47 @@ define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr4
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr5
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY $vgpr6
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[ADD:%[0-9]+]]:vgpr_32(i32) = G_ADD [[COPY6]], [[COPY8]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x s16>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY5]](<2 x s16>)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[LSHR1:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST1]], [[COPY10]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128(<4 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[BITCAST1]](s32), %subreg.sub2, [[LSHR1]](s32), %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 [[COPY6]], [[COPY8]], 0, implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY10]], [[COPY5]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY]](i32), implicit $exec
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY1]](i32), implicit $exec
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY2]](i32), implicit $exec
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY3]](i32), implicit $exec
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[V_READFIRSTLANE_B32_]](i32), %subreg.sub0, [[V_READFIRSTLANE_B32_1]](i32), %subreg.sub1, [[V_READFIRSTLANE_B32_2]](i32), %subreg.sub2, [[V_READFIRSTLANE_B32_3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE1]].sub0_sub1(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE1]].sub2_sub3(<4 x i32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
- ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE]](<4 x i32>), [[ADD]](i32), [[REG_SEQUENCE1]](<4 x i32>), [[COPY7]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll
index 66ea9394f7501..14119b3a31d98 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
define amdgpu_ps void @raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16(ptr addrspace(8) inreg %rsrc, half %val, i32 %voffset, i32 inreg %soffset) {
@@ -73,20 +73,19 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY7]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY]](i32), %subreg.sub0, [[COPY1]](i32), %subreg.sub1, [[COPY2]](i32), %subreg.sub2, [[COPY3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[COPY5]](i32), [[REG_SEQUENCE1]](<4 x i32>), [[COPY6]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16
@@ -112,24 +111,22 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x s16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY8]](s32)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY5]](<2 x s16>)
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR1:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128(<4 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[BITCAST1]](s32), %subreg.sub2, [[LSHR1]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY]](i32), %subreg.sub0, [[COPY1]](i32), %subreg.sub1, [[COPY2]](i32), %subreg.sub2, [[COPY3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE]](<4 x i32>), [[COPY6]](i32), [[REG_SEQUENCE1]](<4 x i32>), [[COPY7]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY5]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
@@ -159,47 +156,44 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr4
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr5
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr6
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x s16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY8]](s32)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY5]](<2 x s16>)
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR1:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128(<4 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[BITCAST1]](s32), %subreg.sub2, [[LSHR1]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128(<4 x i32>) = REG_SEQUENCE [[COPY]](i32), %subreg.sub0, [[COPY1]](i32), %subreg.sub1, [[COPY2]](i32), %subreg.sub2, [[COPY3]](i32), %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY5]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY]](i32), implicit $exec
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY1]](i32), implicit $exec
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY2]](i32), implicit $exec
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY3]](i32), implicit $exec
- ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[V_READFIRSTLANE_B32_]](i32), %subreg.sub0, [[V_READFIRSTLANE_B32_1]](i32), %subreg.sub1, [[V_READFIRSTLANE_B32_2]](i32), %subreg.sub2, [[V_READFIRSTLANE_B32_3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[REG_SEQUENCE1]].sub0_sub1(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[REG_SEQUENCE1]].sub2_sub3(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE2]].sub0_sub1(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE2]].sub2_sub3(<4 x i32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
- ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY14]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE]](<4 x i32>), [[COPY6]](i32), [[REG_SEQUENCE2]](<4 x i32>), [[COPY7]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
@@ -268,20 +262,19 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sreg_32(i32) = G_CONSTANT i32 4095
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x f16>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY6]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY]](i32), %subreg.sub0, [[COPY1]](i32), %subreg.sub1, [[COPY2]](i32), %subreg.sub2, [[COPY3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[COPY5]](i32), [[REG_SEQUENCE1]](<4 x i32>), [[C]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY6]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]], [[COPY5]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095
@@ -307,20 +300,19 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sreg_32(i32) = G_CONSTANT i32 4096
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x f16>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY6]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY]](i32), %subreg.sub0, [[COPY1]](i32), %subreg.sub1, [[COPY2]](i32), %subreg.sub2, [[COPY3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[COPY5]](i32), [[REG_SEQUENCE1]](<4 x i32>), [[C]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY6]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]], [[COPY5]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096
@@ -346,20 +338,19 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY7]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY]](i32), %subreg.sub0, [[COPY1]](i32), %subreg.sub1, [[COPY2]](i32), %subreg.sub2, [[COPY3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[COPY5]](i32), [[REG_SEQUENCE1]](<4 x i32>), [[COPY6]](i32), 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_16
@@ -386,20 +377,19 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY7]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY]](i32), %subreg.sub0, [[COPY1]](i32), %subreg.sub1, [[COPY2]](i32), %subreg.sub2, [[COPY3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[COPY5]](i32), [[REG_SEQUENCE1]](<4 x i32>), [[COPY6]](i32), 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4095
@@ -426,23 +416,22 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[ADD:%[0-9]+]]:vgpr_32(i32) = G_ADD [[COPY5]], [[COPY7]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x f16>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY8]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY]](i32), %subreg.sub0, [[COPY1]](i32), %subreg.sub1, [[COPY2]](i32), %subreg.sub2, [[COPY3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[ADD]](i32), [[REG_SEQUENCE1]](<4 x i32>), [[COPY6]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE1]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4096
@@ -475,50 +464,47 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr4
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr5
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY $vgpr6
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 4096
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[ADD:%[0-9]+]]:vgpr_32(i32) = G_ADD [[COPY6]], [[COPY8]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY4]](<2 x s16>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY5]](<2 x s16>)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[LSHR1:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST1]], [[COPY10]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128(<4 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[BITCAST1]](s32), %subreg.sub2, [[LSHR1]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128(<4 x i32>) = REG_SEQUENCE [[COPY]](i32), %subreg.sub0, [[COPY1]](i32), %subreg.sub1, [[COPY2]](i32), %subreg.sub2, [[COPY3]](i32), %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 [[COPY6]], [[COPY8]], 0, implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY4]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY10]], [[COPY5]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY]](i32), implicit $exec
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY1]](i32), implicit $exec
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY2]](i32), implicit $exec
- ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY3]](i32), implicit $exec
- ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[V_READFIRSTLANE_B32_]](i32), %subreg.sub0, [[V_READFIRSTLANE_B32_1]](i32), %subreg.sub1, [[V_READFIRSTLANE_B32_2]](i32), %subreg.sub2, [[V_READFIRSTLANE_B32_3]](i32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[REG_SEQUENCE1]].sub0_sub1(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[REG_SEQUENCE1]].sub2_sub3(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE2]].sub0_sub1(<4 x i32>)
- ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE2]].sub2_sub3(<4 x i32>)
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
- ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
- ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE]](<4 x i32>), [[ADD]](i32), [[REG_SEQUENCE2]](<4 x i32>), [[COPY7]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.f16.ll
index 3baa6598482c3..fe68540950c6a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.f16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(half %val, ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
@@ -41,20 +41,19 @@ define amdgpu_ps void @raw_tbuffer_store_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY7]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY1]](i32), %subreg.sub0, [[COPY2]](i32), %subreg.sub1, [[COPY3]](i32), %subreg.sub2, [[COPY4]](i32), %subreg.sub3
- ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[COPY5]](i32), [[REG_SEQUENCE1]](<4 x i32>), [[COPY6]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -80,22 +79,20 @@ define amdgpu_ps void @raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY2]](s32)
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(s32) = COPY [[COPY1]](<2 x f16>)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96(<3 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[COPY3]](s32), %subreg.sub2
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY4]](i32), %subreg.sub0, [[COPY5]](i32), %subreg.sub1, [[COPY6]](i32), %subreg.sub2, [[COPY7]](i32), %subreg.sub3
- ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XYZ_gfx80_OFFEN_exact [[REG_SEQUENCE]](<3 x i32>), [[COPY8]](i32), [[REG_SEQUENCE1]](<4 x i32>), [[COPY9]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY2]], [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY1]], %subreg.sub2
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY6]], %subreg.sub3
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XYZ_gfx80_OFFEN_exact [[REG_SEQUENCE]], [[COPY7]], [[REG_SEQUENCE1]], [[COPY8]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -123,24 +120,22 @@ define amdgpu_ps void @raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY8]](s32)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR1:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128(<4 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[BITCAST1]](s32), %subreg.sub2, [[LSHR1]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY2]](i32), %subreg.sub0, [[COPY3]](i32), %subreg.sub1, [[COPY4]](i32), %subreg.sub2, [[COPY5]](i32), %subreg.sub3
- ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE]](<4 x i32>), [[COPY6]](i32), [[REG_SEQUENCE1]](<4 x i32>), [[COPY7]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
index 394df5c6542bb..69a9fb6639c17 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
@@ -59,20 +59,19 @@ define amdgpu_ps void @raw_tbuffer_store_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr_128(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY7]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE]](<2 x i32>), [[COPY5]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -113,22 +112,20 @@ define amdgpu_ps void @raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY2]](s32)
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(s32) = COPY [[COPY1]](<2 x f16>)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY4]](i32), %subreg.sub0, [[COPY5]](i32), %subreg.sub1, [[COPY6]](i32), %subreg.sub2, [[COPY7]](i32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96(<3 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[COPY3]](s32), %subreg.sub2
- ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XYZ_gfx80_OFFEN_exact [[REG_SEQUENCE1]](<3 x i32>), [[COPY8]](i32), [[REG_SEQUENCE]](<4 x i32>), [[COPY9]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY2]], [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY6]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY1]], %subreg.sub2
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XYZ_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -173,24 +170,22 @@ define amdgpu_ps void @raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr_128(<4 x i32>) = G_BUILD_VECTOR [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32)
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY8]](s32)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR1:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128(<4 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[BITCAST1]](s32), %subreg.sub2, [[LSHR1]](s32), %subreg.sub3
- ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE]](<4 x i32>), [[COPY6]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY7]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+ ; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
index 116e641493ac3..7bc61052daa1e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
@@ -65,22 +65,21 @@ define amdgpu_ps void @struct_buffer_store_format_v2f16__vgpr_val__sgpr_rsrc__vg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr_128(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY8]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY5]](i32), %subreg.sub0, [[COPY6]](i32), %subreg.sub1
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_BOTHEN_exact [[REG_SEQUENCE]](<2 x i32>), [[REG_SEQUENCE1]](i64), [[BUILD_VECTOR]](<4 x i32>), [[COPY7]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_BOTHEN_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_buffer_store_format_v2f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -125,24 +124,22 @@ define amdgpu_ps void @struct_buffer_store_format_v3f16__vgpr_val__sgpr_rsrc__vg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY2]](s32)
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(s32) = COPY [[COPY1]](<2 x f16>)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY4]](i32), %subreg.sub0, [[COPY5]](i32), %subreg.sub1, [[COPY6]](i32), %subreg.sub2, [[COPY7]](i32), %subreg.sub3
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96(<3 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[COPY3]](s32), %subreg.sub2
- ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY8]](i32), %subreg.sub0, [[COPY9]](i32), %subreg.sub1
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZ_gfx80_BOTHEN_exact [[REG_SEQUENCE1]](<3 x i32>), [[REG_SEQUENCE2]](i64), [[REG_SEQUENCE]](<4 x i32>), [[COPY10]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY2]], [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY6]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY1]], %subreg.sub2
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZ_gfx80_BOTHEN_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_buffer_store_format_v3f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -191,26 +188,24 @@ define amdgpu_ps void @struct_buffer_store_format_v4f16__vgpr_val__sgpr_rsrc__vg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr_128(<4 x i32>) = G_BUILD_VECTOR [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32)
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR1:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST1]], [[COPY10]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128(<4 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[BITCAST1]](s32), %subreg.sub2, [[LSHR1]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY6]](i32), %subreg.sub0, [[COPY7]](i32), %subreg.sub1
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_BOTHEN_exact [[REG_SEQUENCE]](<4 x i32>), [[REG_SEQUENCE1]](i64), [[BUILD_VECTOR]](<4 x i32>), [[COPY8]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY10]], [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_BOTHEN_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_buffer_store_format_v4f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f16.ll
index 9f60709bd0b9d..28c08d60077ed 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
define amdgpu_ps void @struct_ptr_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(half %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
@@ -45,22 +45,21 @@ define amdgpu_ps void @struct_ptr_buffer_store_format_v2f16__vgpr_val__sgpr_rsrc
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY8]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64(<2 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY1]](i32), %subreg.sub0, [[COPY2]](i32), %subreg.sub1, [[COPY3]](i32), %subreg.sub2, [[COPY4]](i32), %subreg.sub3
- ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY5]](i32), %subreg.sub0, [[COPY6]](i32), %subreg.sub1
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_BOTHEN_exact [[REG_SEQUENCE]](<2 x i32>), [[REG_SEQUENCE2]](i64), [[REG_SEQUENCE1]](<4 x i32>), [[COPY7]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XY_gfx80_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_ptr_buffer_store_format_v2f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -88,24 +87,22 @@ define amdgpu_ps void @struct_ptr_buffer_store_format_v3f16__vgpr_val__sgpr_rsrc
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY2]](s32)
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(s32) = COPY [[COPY1]](<2 x f16>)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96(<3 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[COPY3]](s32), %subreg.sub2
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY4]](i32), %subreg.sub0, [[COPY5]](i32), %subreg.sub1, [[COPY6]](i32), %subreg.sub2, [[COPY7]](i32), %subreg.sub3
- ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY8]](i32), %subreg.sub0, [[COPY9]](i32), %subreg.sub1
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZ_gfx80_BOTHEN_exact [[REG_SEQUENCE]](<3 x i32>), [[REG_SEQUENCE2]](i64), [[REG_SEQUENCE1]](<4 x i32>), [[COPY10]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY2]], [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY1]], %subreg.sub2
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY6]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZ_gfx80_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_ptr_buffer_store_format_v3f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -135,26 +132,24 @@ define amdgpu_ps void @struct_ptr_buffer_store_format_v4f16__vgpr_val__sgpr_rsrc
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[LSHR1:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST1]], [[COPY10]](s32)
- ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128(<4 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[BITCAST1]](s32), %subreg.sub2, [[LSHR1]](s32), %subreg.sub3
- ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY2]](i32), %subreg.sub0, [[COPY3]](i32), %subreg.sub1, [[COPY4]](i32), %subreg.sub2, [[COPY5]](i32), %subreg.sub3
- ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY6]](i32), %subreg.sub0, [[COPY7]](i32), %subreg.sub1
- ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_BOTHEN_exact [[REG_SEQUENCE]](<4 x i32>), [[REG_SEQUENCE2]](i64), [[REG_SEQUENCE1]](<4 x i32>), [[COPY8]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>) into %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY10]], [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+ ; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZW_gfx80_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_ptr_buffer_store_format_v4f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index 3fff6c7c773d0..31ae2405343ef 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -370,27 +370,37 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX9-LABEL: v_saddsat_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff00, v1
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
-; GFX9-NEXT: v_add_i16 v2, v3, v2 clamp
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff00, v2
-; GFX9-NEXT: v_add_i16 v0, v0, v1 clamp
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_add_i16 v0, v0, v1 clamp
+; GFX9-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX9-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_saddsat_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffffff00, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_add_nc_i16 v2, v3, v2 clamp
-; GFX10-NEXT: v_add_nc_i16 v0, v0, v1 clamp
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffffff00, v2
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_add_i16 v0, v0, v1 clamp
+; GFX10-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX10-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_saddsat_v2i8:
@@ -413,14 +423,20 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX11-FAKE16-LABEL: v_saddsat_v2i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffffff00, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_i16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX11-FAKE16-NEXT: v_add_nc_i16 v2, v3, v2 clamp
-; GFX11-FAKE16-NEXT: v_add_nc_i16 v0, v0, v1 clamp
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffffff00, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 8, v0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%lhs = bitcast i16 %lhs.arg to <2 x i8>
@@ -505,71 +521,62 @@ define amdgpu_ps i16 @s_saddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
;
; GFX9-LABEL: s_saddsat_v2i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_lshl_b32 s2, s1, 8
-; GFX9-NEXT: s_and_b32 s1, s1, 0xff00
-; GFX9-NEXT: s_lshl_b32 s3, s0, 8
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: s_and_b32 s0, s0, 0xff00
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_add_i16 v0, s3, v0 clamp
-; GFX9-NEXT: v_add_i16 v1, s0, v1 clamp
-; GFX9-NEXT: s_mov_b32 s0, 0xc0c0105
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_lshr_b32 s3, s1, 8
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX9-NEXT: s_lshl_b32 s2, s2, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_lshr_b32 s2, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX9-NEXT: s_lshl_b32 s2, s2, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_pk_add_i16 v0, s0, v0 clamp
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_sext_i32_i16 s1, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: s_sext_i32_i16 s2, 0x80008
+; GFX9-NEXT: s_ashr_i32 s1, s1, s2
+; GFX9-NEXT: s_ashr_i32 s0, s0, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_and_b32 s0, s0, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: s_saddsat_v2i8:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshl_b32 s2, s1, 8
-; GFX10-NEXT: s_lshl_b32 s3, s0, 8
-; GFX10-NEXT: s_and_b32 s1, s1, 0xff00
-; GFX10-NEXT: s_and_b32 s0, s0, 0xff00
-; GFX10-NEXT: v_add_nc_i16 v0, s3, s2 clamp
-; GFX10-NEXT: v_add_nc_i16 v1, s0, s1 clamp
-; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0105
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: s_saddsat_v2i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x80008
-; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 8
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x80008
-; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
-; GFX11-TRUE16-NEXT: v_pk_add_i16 v0, s0, s1 clamp
-; GFX11-TRUE16-NEXT: s_sext_i32_i16 s1, 0x80008
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: s_sext_i32_i16 s2, s0
-; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 16
-; GFX11-TRUE16-NEXT: s_ashr_i32 s1, s2, s1
-; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 8
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: s_saddsat_v2i8:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s1, 8
-; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s0, 8
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff00
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff00
-; GFX11-FAKE16-NEXT: v_add_nc_i16 v0, s3, s2 clamp
-; GFX11-FAKE16-NEXT: v_add_nc_i16 v1, s0, s1 clamp
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0105
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX10PLUS-LABEL: s_saddsat_v2i8:
+; GFX10PLUS: ; %bb.0:
+; GFX10PLUS-NEXT: s_lshr_b32 s2, s0, 8
+; GFX10PLUS-NEXT: s_lshr_b32 s3, s1, 8
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX10PLUS-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX10PLUS-NEXT: s_lshl_b32 s2, s2, 8
+; GFX10PLUS-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX10PLUS-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX10PLUS-NEXT: v_pk_add_i16 v0, s0, s1 clamp
+; GFX10PLUS-NEXT: s_sext_i32_i16 s1, 0x80008
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10PLUS-NEXT: s_sext_i32_i16 s2, s0
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s2, s1
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 8
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX10PLUS-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10PLUS-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10PLUS-NEXT: s_lshl_b32 s1, s1, 8
+; GFX10PLUS-NEXT: s_or_b32 s0, s0, s1
+; GFX10PLUS-NEXT: ; return to shader part epilog
%lhs = bitcast i16 %lhs.arg to <2 x i8>
%rhs = bitcast i16 %rhs.arg to <2 x i8>
%result = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %lhs, <2 x i8> %rhs)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
index b117baee09b18..344b4ae1c7849 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 %s -o - | FileCheck -check-prefixes=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 %s -o - | FileCheck -check-prefixes=GCN %s
define half @test_s16(half %a) #0 {
; GCN-LABEL: test_s16:
@@ -45,19 +45,21 @@ define <4 x half> @test_v4s16(<4 x half> %a) #0 {
; GCN-LABEL: test_v4s16:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GCN-NEXT: v_mov_b32_e32 v4, 0
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v3
-; GCN-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v2
-; GCN-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v1
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: s_mov_b32 s4, 0x5040100
-; GCN-NEXT: v_perm_b32 v0, v2, v0, s4
-; GCN-NEXT: v_perm_b32 v1, v3, v1, s4
+; GCN-NEXT: v_cndmask_b32_e64 v5, v0, 0, vcc
+; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v0, v4 src0_sel:WORD_1 src1_sel:DWORD
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v1
+; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v2, v1, 0, vcc
+; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v1, v4 src0_sel:WORD_1 src1_sel:DWORD
+; GCN-NEXT: v_cndmask_b32_e64 v1, v3, 0, s[4:5]
+; GCN-NEXT: v_and_b32_e32 v3, 0xffff, v5
+; GCN-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT: v_lshl_or_b32 v0, v0, 16, v3
+; GCN-NEXT: v_lshl_or_b32 v1, v1, 16, v2
; GCN-NEXT: s_setpc_b64 s[30:31]
entry:
%fcmp = fcmp olt <4 x half> %a, zeroinitializer
@@ -69,31 +71,35 @@ define <8 x half> @test_v8s16(<8 x half> %a) #0 {
; GCN-LABEL: test_v8s16:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v3
-; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v7
-; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GCN-NEXT: v_cndmask_b32_e32 v7, 0, v7, vcc
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v6
+; GCN-NEXT: v_mov_b32_e32 v8, 0
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GCN-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v5
-; GCN-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v4
-; GCN-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v3
-; GCN-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v2
-; GCN-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v1
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: s_mov_b32 s4, 0x5040100
-; GCN-NEXT: v_perm_b32 v0, v4, v0, s4
-; GCN-NEXT: v_perm_b32 v1, v5, v1, s4
-; GCN-NEXT: v_perm_b32 v2, v6, v2, s4
-; GCN-NEXT: v_perm_b32 v3, v7, v3, s4
+; GCN-NEXT: v_cndmask_b32_e64 v9, v0, 0, vcc
+; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v0, v8 src0_sel:WORD_1 src1_sel:DWORD
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v1
+; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, v4, 0, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v4, v1, 0, vcc
+; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v1, v8 src0_sel:WORD_1 src1_sel:DWORD
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v2
+; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GCN-NEXT: v_cndmask_b32_e64 v1, v5, 0, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v5, v2, 0, vcc
+; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v2, v8 src0_sel:WORD_1 src1_sel:DWORD
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v3
+; GCN-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GCN-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v6, v3, 0, vcc
+; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v3, v8 src0_sel:WORD_1 src1_sel:DWORD
+; GCN-NEXT: v_lshl_or_b32 v1, v1, 16, v4
+; GCN-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GCN-NEXT: v_cndmask_b32_e64 v3, v7, 0, s[4:5]
+; GCN-NEXT: v_and_b32_e32 v7, 0xffff, v9
+; GCN-NEXT: v_lshl_or_b32 v2, v2, 16, v4
+; GCN-NEXT: v_and_b32_e32 v4, 0xffff, v6
+; GCN-NEXT: v_lshl_or_b32 v0, v0, 16, v7
+; GCN-NEXT: v_lshl_or_b32 v3, v3, 16, v4
; GCN-NEXT: s_setpc_b64 s[30:31]
entry:
%fcmp = fcmp olt <8 x half> %a, zeroinitializer
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
index d21a18f6b96f1..042c56ec364b2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
; Test optimization to reduce shifts to narrower sizes.
@@ -637,10 +637,12 @@ define i32 @v_shl_i32_zext_i16(i16 %x) {
define amdgpu_ps <2 x i32> @s_shl_v2i32_zext_v2i16(<2 x i16> inreg %x) {
; GFX7-LABEL: s_shl_v2i32_zext_v2i16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_lshr_b32 s1, s0, 14
+; GFX7-NEXT: s_and_b32 s0, s0, 0x3fff3fff
+; GFX7-NEXT: s_lshr_b32 s1, s0, 16
; GFX7-NEXT: s_lshl_b32 s0, s0, 2
-; GFX7-NEXT: s_and_b32 s1, s1, 0xfffc
-; GFX7-NEXT: s_and_b32 s0, s0, 0xfffc
+; GFX7-NEXT: s_lshl_b32 s1, s1, 2
+; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX7-NEXT: s_and_b32 s1, s1, 0xffff
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_shl_v2i32_zext_v2i16:
@@ -655,18 +657,24 @@ define amdgpu_ps <2 x i32> @s_shl_v2i32_zext_v2i16(<2 x i16> inreg %x) {
;
; GFX9-LABEL: s_shl_v2i32_zext_v2i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s1, s0, 0x3fff3fff
-; GFX9-NEXT: s_and_b32 s0, s1, 0x3fff
-; GFX9-NEXT: s_lshl_b32 s0, s0, 2
-; GFX9-NEXT: s_lshr_b32 s1, s1, 14
+; GFX9-NEXT: s_and_b32 s0, s0, 0x3fff3fff
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x20002
+; GFX9-NEXT: s_lshl_b32 s1, s1, 2
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_shl_v2i32_zext_v2i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s1, s0, 0x3fff3fff
-; GFX10-NEXT: s_and_b32 s0, s1, 0x3fff
-; GFX10-NEXT: s_lshr_b32 s1, s1, 14
-; GFX10-NEXT: s_lshl_b32 s0, s0, 2
+; GFX10-NEXT: s_and_b32 s0, s0, 0x3fff3fff
+; GFX10-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10-NEXT: s_lshl_b32 s0, s0, 0x20002
+; GFX10-NEXT: s_lshl_b32 s1, s1, 2
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s0, s1
+; GFX10-NEXT: s_and_b32 s0, s1, 0xffff
+; GFX10-NEXT: s_lshr_b32 s1, s1, 16
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_shl_v2i32_zext_v2i16:
@@ -682,10 +690,13 @@ define amdgpu_ps <2 x i32> @s_shl_v2i32_zext_v2i16(<2 x i16> inreg %x) {
;
; GFX11-FAKE16-LABEL: s_shl_v2i32_zext_v2i16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s0, 0x3fff3fff
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s1, 0x3fff
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 14
-; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 2
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x3fff3fff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 0x20002
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 2
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s0, s1
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s1, 0xffff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%and = and <2 x i16> %x, <i16 16383, i16 16383>
%ext = zext <2 x i16> %and to <2 x i32>
@@ -702,38 +713,39 @@ define <2 x i32> @v_shl_v2i32_zext_v2i16(<2 x i16> %x) {
; GFX7-LABEL: v_shl_v2i32_zext_v2i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 2, v0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xfffc, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 14, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xfffc, v0
-; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_and_b32_e32 v0, 0x3fff3fff, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 2, v1
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_shl_v2i32_zext_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v0, 0x3fff3fff, v0
-; GFX8-NEXT: v_and_b32_e32 v2, 0x3fff, v0
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 14, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 2, v2
+; GFX8-NEXT: v_and_b32_e32 v1, 0x3fff3fff, v0
+; GFX8-NEXT: v_mov_b32_e32 v2, 2
+; GFX8-NEXT: v_lshlrev_b16_e32 v0, 2, v1
+; GFX8-NEXT: v_lshlrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_shl_v2i32_zext_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, 0x3fff3fff, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0x3fff, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 14, v0
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v2
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, 2, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_shl_v2i32_zext_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v1, 0x3fff3fff, v0
-; GFX10-NEXT: v_and_b32_e32 v0, 0x3fff, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 14, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-NEXT: v_and_b32_e32 v0, 0x3fff3fff, v0
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, 2, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_shl_v2i32_zext_v2i16:
@@ -748,10 +760,10 @@ define <2 x i32> @v_shl_v2i32_zext_v2i16(<2 x i16> %x) {
; GFX11-FAKE16-LABEL: v_shl_v2i32_zext_v2i16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0x3fff3fff, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3fff, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 14, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3fff3fff, v0
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v1, 2, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%and = and <2 x i16> %x, <i16 16383, i16 16383>
%ext = zext <2 x i16> %and to <2 x i32>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
index 9561a79111635..d99e3832deb46 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-mesa3d -mcpu=gfx802 < %s | FileCheck -check-prefixes=GFX89,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX89,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX89,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
define i32 @test_min_max_ValK0_K1_i32(i32 %a) {
; GFX89-LABEL: test_min_max_ValK0_K1_i32:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index 4f33339eac949..86be699d573ae 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -370,27 +370,37 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX9-LABEL: v_ssubsat_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff00, v1
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
-; GFX9-NEXT: v_sub_i16 v2, v3, v2 clamp
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff00, v2
-; GFX9-NEXT: v_sub_i16 v0, v0, v1 clamp
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
+; GFX9-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX9-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_ssubsat_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffffff00, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_sub_nc_i16 v2, v3, v2 clamp
-; GFX10-NEXT: v_sub_nc_i16 v0, v0, v1 clamp
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffffff00, v2
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
+; GFX10-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX10-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_ssubsat_v2i8:
@@ -413,14 +423,20 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX11-FAKE16-LABEL: v_ssubsat_v2i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffffff00, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX11-FAKE16-NEXT: v_sub_nc_i16 v2, v3, v2 clamp
-; GFX11-FAKE16-NEXT: v_sub_nc_i16 v0, v0, v1 clamp
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffffff00, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 8, v0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%lhs = bitcast i16 %lhs.arg to <2 x i8>
@@ -505,71 +521,62 @@ define amdgpu_ps i16 @s_ssubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
;
; GFX9-LABEL: s_ssubsat_v2i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_lshl_b32 s2, s1, 8
-; GFX9-NEXT: s_and_b32 s1, s1, 0xff00
-; GFX9-NEXT: s_lshl_b32 s3, s0, 8
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: s_and_b32 s0, s0, 0xff00
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_sub_i16 v0, s3, v0 clamp
-; GFX9-NEXT: v_sub_i16 v1, s0, v1 clamp
-; GFX9-NEXT: s_mov_b32 s0, 0xc0c0105
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_lshr_b32 s3, s1, 8
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX9-NEXT: s_lshl_b32 s2, s2, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_lshr_b32 s2, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX9-NEXT: s_lshl_b32 s2, s2, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_pk_sub_i16 v0, s0, v0 clamp
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_sext_i32_i16 s1, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: s_sext_i32_i16 s2, 0x80008
+; GFX9-NEXT: s_ashr_i32 s1, s1, s2
+; GFX9-NEXT: s_ashr_i32 s0, s0, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_and_b32 s0, s0, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: s_ssubsat_v2i8:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshl_b32 s2, s1, 8
-; GFX10-NEXT: s_lshl_b32 s3, s0, 8
-; GFX10-NEXT: s_and_b32 s1, s1, 0xff00
-; GFX10-NEXT: s_and_b32 s0, s0, 0xff00
-; GFX10-NEXT: v_sub_nc_i16 v0, s3, s2 clamp
-; GFX10-NEXT: v_sub_nc_i16 v1, s0, s1 clamp
-; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0105
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-TRUE16-LABEL: s_ssubsat_v2i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x80008
-; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 8
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x80008
-; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
-; GFX11-TRUE16-NEXT: v_pk_sub_i16 v0, s0, s1 clamp
-; GFX11-TRUE16-NEXT: s_sext_i32_i16 s1, 0x80008
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: s_sext_i32_i16 s2, s0
-; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 16
-; GFX11-TRUE16-NEXT: s_ashr_i32 s1, s2, s1
-; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 8
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: s_ssubsat_v2i8:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s1, 8
-; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s0, 8
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff00
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff00
-; GFX11-FAKE16-NEXT: v_sub_nc_i16 v0, s3, s2 clamp
-; GFX11-FAKE16-NEXT: v_sub_nc_i16 v1, s0, s1 clamp
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0105
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX10PLUS-LABEL: s_ssubsat_v2i8:
+; GFX10PLUS: ; %bb.0:
+; GFX10PLUS-NEXT: s_lshr_b32 s2, s0, 8
+; GFX10PLUS-NEXT: s_lshr_b32 s3, s1, 8
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX10PLUS-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10PLUS-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10PLUS-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX10PLUS-NEXT: s_lshl_b32 s2, s2, 8
+; GFX10PLUS-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX10PLUS-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX10PLUS-NEXT: v_pk_sub_i16 v0, s0, s1 clamp
+; GFX10PLUS-NEXT: s_sext_i32_i16 s1, 0x80008
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10PLUS-NEXT: s_sext_i32_i16 s2, s0
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 16
+; GFX10PLUS-NEXT: s_ashr_i32 s1, s2, s1
+; GFX10PLUS-NEXT: s_ashr_i32 s0, s0, 8
+; GFX10PLUS-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX10PLUS-NEXT: s_lshr_b32 s1, s0, 16
+; GFX10PLUS-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10PLUS-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10PLUS-NEXT: s_lshl_b32 s1, s1, 8
+; GFX10PLUS-NEXT: s_or_b32 s0, s0, s1
+; GFX10PLUS-NEXT: ; return to shader part epilog
%lhs = bitcast i16 %lhs.arg to <2 x i8>
%rhs = bitcast i16 %rhs.arg to <2 x i8>
%result = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %lhs, <2 x i8> %rhs)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
index f0f9947d034dd..423a163a8baee 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji < %s | FileCheck -check-prefixes=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX900 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX942 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12 %s
+; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX900 %s
+; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX942 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12 %s
define void @v_constained_fma_f16_fpexcept_strict_uni(half inreg %x, half inreg %y, half inreg %z, ptr addrspace(1) %out) #0 {
; GFX8-LABEL: v_constained_fma_f16_fpexcept_strict_uni:
@@ -16,26 +16,6 @@ define void @v_constained_fma_f16_fpexcept_strict_uni(half inreg %x, half inreg
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s17
-; GFX900-NEXT: v_mov_b32_e32 v3, s18
-; GFX900-NEXT: v_fma_f16 v2, s16, v2, v3
-; GFX900-NEXT: global_store_short v[0:1], v2, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s1
-; GFX942-NEXT: v_mov_b32_e32 v3, s2
-; GFX942-NEXT: v_fma_f16 v2, s0, v2, v3
-; GFX942-NEXT: global_store_short v[0:1], v2, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -58,6 +38,24 @@ define void @v_constained_fma_f16_fpexcept_strict_uni(half inreg %x, half inreg
; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s17
+; GFX900-NEXT: v_mov_b32_e32 v3, s18
+; GFX900-NEXT: v_fma_f16 v2, s16, v2, v3
+; GFX900-NEXT: global_store_short v[0:1], v2, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_fma_f16 v2, s0, v2, v3
+; GFX942-NEXT: global_store_short v[0:1], v2, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%val = call half @llvm.experimental.constrained.fma.f16(half %x, half %y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store half %val, ptr addrspace(1) %out
ret void
@@ -72,24 +70,6 @@ define void @v_constained_fma_f16_fpexcept_strict_div(half %x, half %y, half %z,
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_div:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_fma_f16 v0, v0, v1, v2
-; GFX900-NEXT: global_store_short v[3:4], v0, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_div:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, v3
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_fma_f16 v0, v0, v1, v2
-; GFX942-NEXT: global_store_short v[6:7], v0, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -107,6 +87,22 @@ define void @v_constained_fma_f16_fpexcept_strict_div(half %x, half %y, half %z,
; GFX12-NEXT: v_fma_f16 v0.l, v0.l, v1.l, v2.l
; GFX12-NEXT: global_store_b16 v[3:4], v0, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_div:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX900-NEXT: global_store_short v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_div:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX942-NEXT: global_store_short v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%val = call half @llvm.experimental.constrained.fma.f16(half %x, half %y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store half %val, ptr addrspace(1) %out
ret void
@@ -131,26 +127,6 @@ define void @v_constained_fma_v2f16_fpexcept_strict_uni(<2 x half> inreg %x, <2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s17
-; GFX900-NEXT: v_mov_b32_e32 v3, s18
-; GFX900-NEXT: v_pk_fma_f16 v2, s16, v2, v3
-; GFX900-NEXT: global_store_dword v[0:1], v2, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s1
-; GFX942-NEXT: v_mov_b32_e32 v3, s2
-; GFX942-NEXT: v_pk_fma_f16 v2, s0, v2, v3
-; GFX942-NEXT: global_store_dword v[0:1], v2, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_v2f16_fpexcept_strict_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -180,6 +156,24 @@ define void @v_constained_fma_v2f16_fpexcept_strict_uni(<2 x half> inreg %x, <2
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s17
+; GFX900-NEXT: v_mov_b32_e32 v3, s18
+; GFX900-NEXT: v_pk_fma_f16 v2, s16, v2, v3
+; GFX900-NEXT: global_store_dword v[0:1], v2, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_pk_fma_f16 v2, s0, v2, v3
+; GFX942-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%val = call <2 x half> @llvm.experimental.constrained.fma.v2f16(<2 x half> %x, <2 x half> %y, <2 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store <2 x half> %val, ptr addrspace(1) %out
ret void
@@ -200,24 +194,6 @@ define void @v_constained_fma_v2f16_fpexcept_strict_div(<2 x half> %x, <2 x half
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_div:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_pk_fma_f16 v0, v0, v1, v2
-; GFX900-NEXT: global_store_dword v[3:4], v0, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_div:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, v3
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_pk_fma_f16 v0, v0, v1, v2
-; GFX942-NEXT: global_store_dword v[6:7], v0, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_v2f16_fpexcept_strict_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -235,6 +211,22 @@ define void @v_constained_fma_v2f16_fpexcept_strict_div(<2 x half> %x, <2 x half
; GFX12-NEXT: v_pk_fma_f16 v0, v0, v1, v2
; GFX12-NEXT: global_store_b32 v[3:4], v0, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_div:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_pk_fma_f16 v0, v0, v1, v2
+; GFX900-NEXT: global_store_dword v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_div:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-NEXT: v_pk_fma_f16 v0, v0, v1, v2
+; GFX942-NEXT: global_store_dword v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%val = call <2 x half> @llvm.experimental.constrained.fma.v2f16(<2 x half> %x, <2 x half> %y, <2 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store <2 x half> %val, ptr addrspace(1) %out
ret void
@@ -250,32 +242,22 @@ define void @v_constained_fma_v3f16_fpexcept_strict_div(<3 x half> %x, <3 x half
; GFX8-LABEL: v_constained_fma_v3f16_fpexcept_strict_div:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v4
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v0
-; GFX8-NEXT: v_fma_f16 v8, v10, v9, v8
-; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; GFX8-NEXT: v_fma_f16 v0, v0, v2, v4
-; GFX8-NEXT: v_or_b32_e32 v2, v0, v8
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v6
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX8-NEXT: flat_store_short v[6:7], v0
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 2, v6
+; GFX8-NEXT: v_fma_f16 v2, v8, v9, v10
; GFX8-NEXT: v_fma_f16 v3, v1, v3, v5
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX8-NEXT: flat_store_short v[0:1], v2
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v6
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
; GFX8-NEXT: flat_store_short v[0:1], v3
-; GFX8-NEXT: flat_store_dword v[6:7], v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_constained_fma_v3f16_fpexcept_strict_div:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_fma_f16 v0, v0, v2, v4
-; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v5
-; GFX9-NEXT: global_store_short v[6:7], v0, off
-; GFX9-NEXT: global_store_short_d16_hi v[6:7], v0, off offset:2
-; GFX9-NEXT: global_store_short v[6:7], v1, off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_v3f16_fpexcept_strict_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -301,6 +283,16 @@ define void @v_constained_fma_v3f16_fpexcept_strict_div(<3 x half> %x, <3 x half
; GFX12-NEXT: global_store_d16_hi_b16 v[6:7], v0, off offset:2
; GFX12-NEXT: global_store_b16 v[6:7], v1, off offset:4
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: v_constained_fma_v3f16_fpexcept_strict_div:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_pk_fma_f16 v0, v0, v2, v4
+; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v5
+; GFX9-NEXT: global_store_short v[6:7], v0, off
+; GFX9-NEXT: global_store_short_d16_hi v[6:7], v0, off offset:2
+; GFX9-NEXT: global_store_short v[6:7], v1, off offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
%val = call <3 x half> @llvm.experimental.constrained.fma.v3f16(<3 x half> %x, <3 x half> %y, <3 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store <3 x half> %val, ptr addrspace(1) %out
ret void
@@ -336,32 +328,6 @@ define void @v_constained_fma_v4f16_fpexcept_strict_uni(<4 x half> inreg %x, <4
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_v4f16_fpexcept_strict_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s18
-; GFX900-NEXT: v_mov_b32_e32 v3, s20
-; GFX900-NEXT: v_pk_fma_f16 v2, s16, v2, v3
-; GFX900-NEXT: v_mov_b32_e32 v3, s19
-; GFX900-NEXT: v_mov_b32_e32 v4, s21
-; GFX900-NEXT: v_pk_fma_f16 v3, s17, v3, v4
-; GFX900-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_v4f16_fpexcept_strict_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s2
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
-; GFX942-NEXT: v_pk_fma_f16 v2, s0, v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, s3
-; GFX942-NEXT: v_mov_b32_e32 v4, s17
-; GFX942-NEXT: v_pk_fma_f16 v3, s1, v3, v4
-; GFX942-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_v4f16_fpexcept_strict_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -399,6 +365,30 @@ define void @v_constained_fma_v4f16_fpexcept_strict_uni(<4 x half> inreg %x, <4
; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_v4f16_fpexcept_strict_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s18
+; GFX900-NEXT: v_mov_b32_e32 v3, s20
+; GFX900-NEXT: v_pk_fma_f16 v2, s16, v2, v3
+; GFX900-NEXT: v_mov_b32_e32 v3, s19
+; GFX900-NEXT: v_mov_b32_e32 v4, s21
+; GFX900-NEXT: v_pk_fma_f16 v3, s17, v3, v4
+; GFX900-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_v4f16_fpexcept_strict_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s2
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
+; GFX942-NEXT: v_pk_fma_f16 v2, s0, v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, s3
+; GFX942-NEXT: v_mov_b32_e32 v4, s17
+; GFX942-NEXT: v_pk_fma_f16 v3, s1, v3, v4
+; GFX942-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%val = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> %x, <4 x half> %y, <4 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store <4 x half> %val, ptr addrspace(1) %out
ret void
@@ -426,15 +416,6 @@ define void @v_constained_fma_v4f16_fpexcept_strict_div(<4 x half> %x, <4 x half
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_constained_fma_v4f16_fpexcept_strict_div:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_fma_f16 v0, v0, v2, v4
-; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v5
-; GFX9-NEXT: global_store_dwordx2 v[6:7], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_v4f16_fpexcept_strict_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -454,6 +435,14 @@ define void @v_constained_fma_v4f16_fpexcept_strict_div(<4 x half> %x, <4 x half
; GFX12-NEXT: v_pk_fma_f16 v1, v1, v3, v5
; GFX12-NEXT: global_store_b64 v[6:7], v[0:1], off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: v_constained_fma_v4f16_fpexcept_strict_div:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_pk_fma_f16 v0, v0, v2, v4
+; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v5
+; GFX9-NEXT: global_store_dwordx2 v[6:7], v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
%val = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> %x, <4 x half> %y, <4 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store <4 x half> %val, ptr addrspace(1) %out
ret void
@@ -470,26 +459,6 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s16
-; GFX900-NEXT: v_mov_b32_e32 v3, s17
-; GFX900-NEXT: v_fma_f16 v2, v2, v3, -s18
-; GFX900-NEXT: global_store_short v[0:1], v2, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-NEXT: v_mov_b32_e32 v3, s1
-; GFX942-NEXT: v_fma_f16 v2, v2, v3, -s2
-; GFX942-NEXT: global_store_short v[0:1], v2, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -514,6 +483,24 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s16
+; GFX900-NEXT: v_mov_b32_e32 v3, s17
+; GFX900-NEXT: v_fma_f16 v2, v2, v3, -s18
+; GFX900-NEXT: global_store_short v[0:1], v2, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s0
+; GFX942-NEXT: v_mov_b32_e32 v3, s1
+; GFX942-NEXT: v_fma_f16 v2, v2, v3, -s2
+; GFX942-NEXT: global_store_short v[0:1], v2, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.z = fneg half %z
%val = call half @llvm.experimental.constrained.fma.f16(half %x, half %y, half %neg.z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store half %val, ptr addrspace(1) %out
@@ -529,24 +516,6 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_div(half %x, half %y, hal
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_div:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_fma_f16 v0, v0, v1, -v2
-; GFX900-NEXT: global_store_short v[3:4], v0, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_div:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, v3
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_fma_f16 v0, v0, v1, -v2
-; GFX942-NEXT: global_store_short v[6:7], v0, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -564,6 +533,22 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_div(half %x, half %y, hal
; GFX12-NEXT: v_fma_f16 v0.l, v0.l, v1.l, -v2.l
; GFX12-NEXT: global_store_b16 v[3:4], v0, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_div:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_fma_f16 v0, v0, v1, -v2
+; GFX900-NEXT: global_store_short v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_div:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-NEXT: v_fma_f16 v0, v0, v1, -v2
+; GFX942-NEXT: global_store_short v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.z = fneg half %z
%val = call half @llvm.experimental.constrained.fma.f16(half %x, half %y, half %neg.z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store half %val, ptr addrspace(1) %out
@@ -581,26 +566,6 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni(half inreg %x, h
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s17
-; GFX900-NEXT: v_mov_b32_e32 v3, s18
-; GFX900-NEXT: v_fma_f16 v2, -s16, -v2, v3
-; GFX900-NEXT: global_store_short v[0:1], v2, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s1
-; GFX942-NEXT: v_mov_b32_e32 v3, s2
-; GFX942-NEXT: v_fma_f16 v2, -s0, -v2, v3
-; GFX942-NEXT: global_store_short v[0:1], v2, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -626,6 +591,24 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni(half inreg %x, h
; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s17
+; GFX900-NEXT: v_mov_b32_e32 v3, s18
+; GFX900-NEXT: v_fma_f16 v2, -s16, -v2, v3
+; GFX900-NEXT: global_store_short v[0:1], v2, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_fma_f16 v2, -s0, -v2, v3
+; GFX942-NEXT: global_store_short v[0:1], v2, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.x = fneg half %x
%neg.y = fneg half %y
%val = call half @llvm.experimental.constrained.fma.f16(half %neg.x, half %neg.y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -642,24 +625,6 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_fneg_div(half %x, half %y
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_div:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_fma_f16 v0, -v0, -v1, v2
-; GFX900-NEXT: global_store_short v[3:4], v0, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_div:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, v3
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_fma_f16 v0, -v0, -v1, v2
-; GFX942-NEXT: global_store_short v[6:7], v0, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -677,6 +642,22 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_fneg_div(half %x, half %y
; GFX12-NEXT: v_fma_f16 v0.l, -v0.l, -v1.l, v2.l
; GFX12-NEXT: global_store_b16 v[3:4], v0, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_div:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_fma_f16 v0, -v0, -v1, v2
+; GFX900-NEXT: global_store_short v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_div:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-NEXT: v_fma_f16 v0, -v0, -v1, v2
+; GFX942-NEXT: global_store_short v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.x = fneg half %x
%neg.y = fneg half %y
%val = call half @llvm.experimental.constrained.fma.f16(half %neg.x, half %neg.y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -695,26 +676,6 @@ define void @v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni(half inreg %x, h
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s17
-; GFX900-NEXT: v_mov_b32_e32 v3, s18
-; GFX900-NEXT: v_fma_f16 v2, |s16|, |v2|, v3
-; GFX900-NEXT: global_store_short v[0:1], v2, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s1
-; GFX942-NEXT: v_mov_b32_e32 v3, s2
-; GFX942-NEXT: v_fma_f16 v2, |s0|, |v2|, v3
-; GFX942-NEXT: global_store_short v[0:1], v2, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -740,6 +701,24 @@ define void @v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni(half inreg %x, h
; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s17
+; GFX900-NEXT: v_mov_b32_e32 v3, s18
+; GFX900-NEXT: v_fma_f16 v2, |s16|, |v2|, v3
+; GFX900-NEXT: global_store_short v[0:1], v2, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_fma_f16 v2, |s0|, |v2|, v3
+; GFX942-NEXT: global_store_short v[0:1], v2, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.x = call half @llvm.fabs.f16(half %x) #0
%neg.y = call half @llvm.fabs.f16(half %y) #0
%val = call half @llvm.experimental.constrained.fma.f16(half %neg.x, half %neg.y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -756,24 +735,6 @@ define void @v_constained_fma_f16_fpexcept_strict_fabs_fabs_div(half %x, half %y
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_div:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_fma_f16 v0, |v0|, |v1|, v2
-; GFX900-NEXT: global_store_short v[3:4], v0, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_div:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, v3
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_fma_f16 v0, |v0|, |v1|, v2
-; GFX942-NEXT: global_store_short v[6:7], v0, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -791,6 +752,22 @@ define void @v_constained_fma_f16_fpexcept_strict_fabs_fabs_div(half %x, half %y
; GFX12-NEXT: v_fma_f16 v0.l, |v0.l|, |v1.l|, v2.l
; GFX12-NEXT: global_store_b16 v[3:4], v0, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_div:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_fma_f16 v0, |v0|, |v1|, v2
+; GFX900-NEXT: global_store_short v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_div:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-NEXT: v_fma_f16 v0, |v0|, |v1|, v2
+; GFX942-NEXT: global_store_short v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.x = call half @llvm.fabs.f16(half %x) #0
%neg.y = call half @llvm.fabs.f16(half %y) #0
%val = call half @llvm.experimental.constrained.fma.f16(half %neg.x, half %neg.y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -817,26 +794,6 @@ define void @v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni(<2 x half> inr
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s17
-; GFX900-NEXT: v_mov_b32_e32 v3, s18
-; GFX900-NEXT: v_pk_fma_f16 v2, s16, v2, v3 neg_lo:[1,1,0] neg_hi:[1,1,0]
-; GFX900-NEXT: global_store_dword v[0:1], v2, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s1
-; GFX942-NEXT: v_mov_b32_e32 v3, s2
-; GFX942-NEXT: v_pk_fma_f16 v2, s0, v2, v3 neg_lo:[1,1,0] neg_hi:[1,1,0]
-; GFX942-NEXT: global_store_dword v[0:1], v2, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -871,6 +828,24 @@ define void @v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni(<2 x half> inr
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s17
+; GFX900-NEXT: v_mov_b32_e32 v3, s18
+; GFX900-NEXT: v_pk_fma_f16 v2, s16, v2, v3 neg_lo:[1,1,0] neg_hi:[1,1,0]
+; GFX900-NEXT: global_store_dword v[0:1], v2, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_pk_fma_f16 v2, s0, v2, v3 neg_lo:[1,1,0] neg_hi:[1,1,0]
+; GFX942-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.x = fneg <2 x half> %x
%neg.y = fneg <2 x half> %y
%val = call <2 x half> @llvm.experimental.constrained.fma.v2f16(<2 x half> %neg.x, <2 x half> %neg.y, <2 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -893,24 +868,6 @@ define void @v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div(<2 x half> %x,
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_pk_fma_f16 v0, v0, v1, v2 neg_lo:[1,1,0] neg_hi:[1,1,0]
-; GFX900-NEXT: global_store_dword v[3:4], v0, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, v3
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_pk_fma_f16 v0, v0, v1, v2 neg_lo:[1,1,0] neg_hi:[1,1,0]
-; GFX942-NEXT: global_store_dword v[6:7], v0, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX11-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -928,6 +885,22 @@ define void @v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div(<2 x half> %x,
; GFX12-NEXT: v_pk_fma_f16 v0, v0, v1, v2 neg_lo:[1,1,0] neg_hi:[1,1,0]
; GFX12-NEXT: global_store_b32 v[3:4], v0, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_pk_fma_f16 v0, v0, v1, v2 neg_lo:[1,1,0] neg_hi:[1,1,0]
+; GFX900-NEXT: global_store_dword v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-NEXT: v_pk_fma_f16 v0, v0, v1, v2 neg_lo:[1,1,0] neg_hi:[1,1,0]
+; GFX942-NEXT: global_store_dword v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.x = fneg <2 x half> %x
%neg.y = fneg <2 x half> %y
%val = call <2 x half> @llvm.experimental.constrained.fma.v2f16(<2 x half> %neg.x, <2 x half> %neg.y, <2 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index a113c6cd10bb5..8f06d7d879f26 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -296,27 +296,37 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX9-LABEL: v_uaddsat_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0xff
-; GFX9-NEXT: v_add_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX9-NEXT: v_min_u16_e32 v2, 0xff, v2
-; GFX9-NEXT: v_min_u16_sdwa v0, v0, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_add_u16 v0, v0, v1 clamp
+; GFX9-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX9-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_uaddsat_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v2, 8, v1
-; GFX10-NEXT: v_lshrrev_b16 v3, 8, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX10-NEXT: v_add_nc_u16 v2, v3, v2
-; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-NEXT: v_min_u16 v1, 0xff, v2
-; GFX10-NEXT: v_min_u16 v0, 0xff, v0
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_add_u16 v0, v0, v1 clamp
+; GFX10-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX10-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_uaddsat_v2i8:
@@ -337,14 +347,19 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX11-FAKE16-LABEL: v_uaddsat_v2i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v2, 8, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v3, 8, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u16 v2, v3, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX11-FAKE16-NEXT: v_min_u16 v1, 0xff, v2
-; GFX11-FAKE16-NEXT: v_min_u16 v0, 0xff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -400,36 +415,57 @@ define amdgpu_ps i16 @s_uaddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
;
; GFX9-LABEL: s_uaddsat_v2i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s2, 0xffff, s1
-; GFX9-NEXT: s_and_b32 s3, 0xffff, s0
-; GFX9-NEXT: s_lshr_b32 s2, s2, 8
-; GFX9-NEXT: s_lshr_b32 s3, s3, 8
+; GFX9-NEXT: s_lshr_b32 s2, s0, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_lshr_b32 s3, s1, 8
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX9-NEXT: s_lshl_b32 s2, s2, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_lshr_b32 s2, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX9-NEXT: s_lshl_b32 s2, s2, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_pk_add_u16 v0, s0, v0 clamp
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_and_b32 s1, s0, 0xffff
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: s_lshr_b32 s1, s1, 0x80008
+; GFX9-NEXT: s_lshr_b32 s0, s0, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
; GFX9-NEXT: s_and_b32 s0, s0, 0xff
-; GFX9-NEXT: s_add_i32 s3, s3, s2
-; GFX9-NEXT: s_add_i32 s0, s0, s1
-; GFX9-NEXT: s_and_b32 s1, s3, 0x1ff
-; GFX9-NEXT: s_and_b32 s0, s0, 0x1ff
-; GFX9-NEXT: s_min_u32 s1, s1, 0xff
-; GFX9-NEXT: s_min_u32 s0, s0, 0xff
; GFX9-NEXT: s_lshl_b32 s1, s1, 8
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_uaddsat_v2i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s2, 0xffff, s1
-; GFX10-NEXT: s_and_b32 s3, 0xffff, s0
-; GFX10-NEXT: s_lshr_b32 s2, s2, 8
-; GFX10-NEXT: s_lshr_b32 s3, s3, 8
-; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshr_b32 s2, s0, 8
+; GFX10-NEXT: s_lshr_b32 s3, s1, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX10-NEXT: s_lshl_b32 s2, s2, 8
+; GFX10-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX10-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT: v_pk_add_u16 v0, s0, s1 clamp
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_and_b32 s1, s0, 0xffff
+; GFX10-NEXT: s_lshr_b32 s0, s0, 16
+; GFX10-NEXT: s_lshr_b32 s1, s1, 0x80008
+; GFX10-NEXT: s_lshr_b32 s0, s0, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX10-NEXT: s_lshr_b32 s1, s0, 16
; GFX10-NEXT: s_and_b32 s0, s0, 0xff
-; GFX10-NEXT: s_add_i32 s3, s3, s2
-; GFX10-NEXT: s_add_i32 s0, s0, s1
-; GFX10-NEXT: s_and_b32 s1, s3, 0x1ff
-; GFX10-NEXT: s_and_b32 s0, s0, 0x1ff
-; GFX10-NEXT: s_min_u32 s1, s1, 0xff
-; GFX10-NEXT: s_min_u32 s0, s0, 0xff
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
; GFX10-NEXT: s_lshl_b32 s1, s1, 8
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
@@ -462,18 +498,28 @@ define amdgpu_ps i16 @s_uaddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
;
; GFX11-FAKE16-LABEL: s_uaddsat_v2i8:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_b32 s2, 0xffff, s1
-; GFX11-FAKE16-NEXT: s_and_b32 s3, 0xffff, s0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 8
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v0, s0, s1 clamp
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s0, 0xffff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 0x80008
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-FAKE16-NEXT: s_add_i32 s3, s3, s2
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, s1
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s3, 0x1ff
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x1ff
-; GFX11-FAKE16-NEXT: s_min_u32 s1, s1, 0xff
-; GFX11-FAKE16-NEXT: s_min_u32 s0, s0, 0xff
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s1
; GFX11-FAKE16-NEXT: ; return to shader part epilog
@@ -1892,16 +1938,20 @@ define <2 x i16> @v_uaddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX6-LABEL: v_uaddsat_v2i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, v3, v2
-; GFX6-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_not_b32_e32 v4, v0
+; GFX6-NEXT: v_min_u32_e32 v1, v4, v1
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT: v_not_b32_e32 v3, v1
+; GFX6-NEXT: v_min_u32_e32 v2, v3, v2
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_uaddsat_v2i16:
@@ -1930,16 +1980,22 @@ define <2 x i16> @v_uaddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
define amdgpu_ps i32 @s_uaddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs) {
; GFX6-LABEL: s_uaddsat_v2i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s2, s1, 16
-; GFX6-NEXT: s_lshr_b32 s3, s0, 16
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: s_add_i32 s3, s3, s2
-; GFX6-NEXT: s_add_i32 s0, s0, s1
-; GFX6-NEXT: s_min_u32 s1, s3, 0xffff
-; GFX6-NEXT: s_min_u32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_lshr_b32 s3, s1, 16
; GFX6-NEXT: s_lshl_b32 s1, s1, 16
-; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_not_b32 s4, s0
+; GFX6-NEXT: s_min_u32 s1, s4, s1
+; GFX6-NEXT: s_add_i32 s0, s0, s1
+; GFX6-NEXT: s_lshl_b32 s1, s2, 16
+; GFX6-NEXT: s_lshl_b32 s2, s3, 16
+; GFX6-NEXT: s_not_b32 s3, s1
+; GFX6-NEXT: s_min_u32 s2, s3, s2
+; GFX6-NEXT: s_add_i32 s1, s1, s2
+; GFX6-NEXT: s_lshr_b32 s1, s1, 16
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_alignbit_b32 v0, s1, v0, 16
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_uaddsat_v2i16:
@@ -1975,16 +2031,20 @@ define amdgpu_ps i32 @s_uaddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
define amdgpu_ps float @uaddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
; GFX6-LABEL: uaddsat_v2i16_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX6-NEXT: s_lshr_b32 s1, s0, 16
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s1, v1
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_not_b32 s2, s0
+; GFX6-NEXT: v_min_u32_e32 v0, s2, v0
; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX6-NEXT: s_lshl_b32 s0, s1, 16
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: s_not_b32 s1, s0
+; GFX6-NEXT: v_min_u32_e32 v1, s1, v1
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, s0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: uaddsat_v2i16_sv:
@@ -2013,16 +2073,20 @@ define amdgpu_ps float @uaddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
define amdgpu_ps float @uaddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
; GFX6-LABEL: uaddsat_v2i16_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s1, s0, 16
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s1, v1
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_lshr_b32 s1, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: v_not_b32_e32 v2, v0
+; GFX6-NEXT: v_min_u32_e32 v2, s0, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT: s_lshl_b32 s0, s1, 16
+; GFX6-NEXT: v_not_b32_e32 v2, v1
+; GFX6-NEXT: v_min_u32_e32 v2, s0, v2
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: uaddsat_v2i16_vs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
index a3435414023b3..3b6f0f9715546 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
@@ -290,22 +290,37 @@ define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX9-LABEL: v_usubsat_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_sub_u16_sdwa v2, v0, v1 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_sub_u16_sdwa v0, v0, v1 clamp dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
+; GFX9-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX9-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_usubsat_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v2, 8, v1
-; GFX10-NEXT: v_lshrrev_b16 v3, 8, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX10-NEXT: v_sub_nc_u16 v2, v3, v2 clamp
-; GFX10-NEXT: v_sub_nc_u16 v0, v0, v1 clamp
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v2
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
+; GFX10-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX10-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_usubsat_v2i8:
@@ -326,13 +341,20 @@ define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX11-FAKE16-LABEL: v_usubsat_v2i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v2, 8, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v3, 8, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: v_sub_nc_u16 v2, v3, v2 clamp
-; GFX11-FAKE16-NEXT: v_sub_nc_u16 v0, v0, v1 clamp
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%lhs = bitcast i16 %lhs.arg to <2 x i8>
@@ -385,33 +407,58 @@ define amdgpu_ps i16 @s_usubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
;
; GFX9-LABEL: s_usubsat_v2i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_and_b32 s2, 0xffff, s1
-; GFX9-NEXT: s_and_b32 s3, 0xffff, s0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_lshr_b32 s3, s1, 8
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX9-NEXT: s_lshl_b32 s2, s2, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_lshr_b32 s2, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX9-NEXT: s_lshl_b32 s2, s2, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_pk_sub_u16 v0, s0, v0 clamp
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_and_b32 s1, s0, 0xffff
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: s_lshr_b32 s1, s1, 0x80008
+; GFX9-NEXT: s_lshr_b32 s0, s0, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
; GFX9-NEXT: s_and_b32 s0, s0, 0xff
-; GFX9-NEXT: s_lshr_b32 s2, s2, 8
-; GFX9-NEXT: s_lshr_b32 s3, s3, 8
-; GFX9-NEXT: s_max_u32 s0, s0, s1
-; GFX9-NEXT: s_sub_i32 s0, s0, s1
-; GFX9-NEXT: s_max_u32 s1, s3, s2
-; GFX9-NEXT: s_sub_i32 s1, s1, s2
; GFX9-NEXT: s_lshl_b32 s1, s1, 8
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_usubsat_v2i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_and_b32 s2, 0xffff, s1
-; GFX10-NEXT: s_and_b32 s3, 0xffff, s0
-; GFX10-NEXT: s_lshr_b32 s2, s2, 8
-; GFX10-NEXT: s_lshr_b32 s3, s3, 8
-; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshr_b32 s2, s0, 8
+; GFX10-NEXT: s_lshr_b32 s3, s1, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX10-NEXT: s_lshl_b32 s2, s2, 8
+; GFX10-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX10-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT: v_pk_sub_u16 v0, s0, s1 clamp
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: s_and_b32 s1, s0, 0xffff
+; GFX10-NEXT: s_lshr_b32 s0, s0, 16
+; GFX10-NEXT: s_lshr_b32 s1, s1, 0x80008
+; GFX10-NEXT: s_lshr_b32 s0, s0, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX10-NEXT: s_lshr_b32 s1, s0, 16
; GFX10-NEXT: s_and_b32 s0, s0, 0xff
-; GFX10-NEXT: s_max_u32 s3, s3, s2
-; GFX10-NEXT: s_max_u32 s0, s0, s1
-; GFX10-NEXT: s_sub_i32 s2, s3, s2
-; GFX10-NEXT: s_sub_i32 s0, s0, s1
-; GFX10-NEXT: s_lshl_b32 s1, s2, 8
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_lshl_b32 s1, s1, 8
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
@@ -443,17 +490,29 @@ define amdgpu_ps i16 @s_usubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
;
; GFX11-FAKE16-LABEL: s_usubsat_v2i8:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_b32 s2, 0xffff, s1
-; GFX11-FAKE16-NEXT: s_and_b32 s3, 0xffff, s0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 8
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s1, s3
+; GFX11-FAKE16-NEXT: v_pk_sub_u16 v0, s0, s1 clamp
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s0, 0xffff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 0x80008
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-FAKE16-NEXT: s_max_u32 s3, s3, s2
-; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, s1
-; GFX11-FAKE16-NEXT: s_sub_i32 s2, s3, s2
-; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, s1
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s2, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s1
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%lhs = bitcast i16 %lhs.arg to <2 x i8>
@@ -1778,16 +1837,18 @@ define <2 x i16> @v_usubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX6-LABEL: v_usubsat_v2i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_min_u32_e32 v1, v0, v1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_max_u32_e32 v1, v3, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT: v_min_u32_e32 v2, v1, v2
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_usubsat_v2i16:
@@ -1816,16 +1877,20 @@ define <2 x i16> @v_usubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
define amdgpu_ps i32 @s_usubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs) {
; GFX6-LABEL: s_usubsat_v2i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s2, s1, 16
-; GFX6-NEXT: s_lshr_b32 s3, s0, 16
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: s_max_u32 s0, s0, s1
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: s_lshr_b32 s3, s1, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_min_u32 s1, s0, s1
; GFX6-NEXT: s_sub_i32 s0, s0, s1
-; GFX6-NEXT: s_max_u32 s1, s3, s2
+; GFX6-NEXT: s_lshl_b32 s1, s2, 16
+; GFX6-NEXT: s_lshl_b32 s2, s3, 16
+; GFX6-NEXT: s_min_u32 s2, s1, s2
; GFX6-NEXT: s_sub_i32 s1, s1, s2
-; GFX6-NEXT: s_lshl_b32 s1, s1, 16
-; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_lshr_b32 s1, s1, 16
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_alignbit_b32 v0, s1, v0, 16
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_usubsat_v2i16:
@@ -1862,16 +1927,18 @@ define amdgpu_ps i32 @s_usubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
define amdgpu_ps float @usubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
; GFX6-LABEL: usubsat_v2i16_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX6-NEXT: s_lshr_b32 s1, s0, 16
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: v_max_u32_e32 v2, s0, v0
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v2, v0
-; GFX6-NEXT: v_max_u32_e32 v2, s1, v1
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v2, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_min_u32_e32 v0, s0, v0
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT: s_lshl_b32 s0, s1, 16
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_min_u32_e32 v1, s0, v1
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: usubsat_v2i16_sv:
@@ -1900,16 +1967,18 @@ define amdgpu_ps float @usubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
define amdgpu_ps float @usubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
; GFX6-LABEL: usubsat_v2i16_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s1, s0, 16
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_max_u32_e32 v1, s1, v1
-; GFX6-NEXT: v_max_u32_e32 v0, s0, v0
-; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, s1, v1
-; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT: s_lshr_b32 s1, s0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: v_min_u32_e32 v2, s0, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: s_lshl_b32 s0, s1, 16
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT: v_min_u32_e32 v2, s0, v1
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: usubsat_v2i16_vs:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index bf5d640f236de..e60f62c26ab5e 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -2281,17 +2281,20 @@ define <6 x i8> @load_v6i8(ptr addrspace(8) inreg %buf) {
; GISEL-LABEL: load_v6i8:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: buffer_load_ushort v6, off, s[16:19], 0 offset:4
; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 0
+; GISEL-NEXT: buffer_load_ushort v4, off, s[16:19], 0 offset:4
; GISEL-NEXT: s_waitcnt vmcnt(1)
-; GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v6
+; GISEL-NEXT: v_readfirstlane_b32 s4, v0
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: v_lshrrev_b32_e32 v7, 8, v0
-; GISEL-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
-; GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GISEL-NEXT: v_mov_b32_e32 v4, v6
-; GISEL-NEXT: v_mov_b32_e32 v1, v7
+; GISEL-NEXT: v_readfirstlane_b32 s5, v4
+; GISEL-NEXT: s_lshr_b32 s6, s4, 8
+; GISEL-NEXT: s_lshr_b32 s7, s4, 16
+; GISEL-NEXT: s_lshr_b32 s4, s4, 24
+; GISEL-NEXT: s_lshr_b32 s5, s5, 8
+; GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GISEL-NEXT: v_mov_b32_e32 v2, s7
+; GISEL-NEXT: v_mov_b32_e32 v3, s4
+; GISEL-NEXT: v_mov_b32_e32 v5, s5
; GISEL-NEXT: s_setpc_b64 s[30:31]
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
%ret = load <6 x i8>, ptr addrspace(7) %p
@@ -3707,15 +3710,18 @@ define <6 x i8> @volatile_load_v6i8(ptr addrspace(8) inreg %buf) {
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 0 glc
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: buffer_load_ushort v6, off, s[16:19], 0 offset:4 glc
+; GISEL-NEXT: buffer_load_ushort v4, off, s[16:19], 0 offset:4 glc
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: v_lshrrev_b32_e32 v7, 8, v0
-; GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v6
-; GISEL-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
-; GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GISEL-NEXT: v_mov_b32_e32 v4, v6
-; GISEL-NEXT: v_mov_b32_e32 v1, v7
+; GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GISEL-NEXT: v_readfirstlane_b32 s5, v4
+; GISEL-NEXT: s_lshr_b32 s6, s4, 8
+; GISEL-NEXT: s_lshr_b32 s7, s4, 16
+; GISEL-NEXT: s_lshr_b32 s4, s4, 24
+; GISEL-NEXT: s_lshr_b32 s5, s5, 8
+; GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GISEL-NEXT: v_mov_b32_e32 v2, s7
+; GISEL-NEXT: v_mov_b32_e32 v3, s4
+; GISEL-NEXT: v_mov_b32_e32 v5, s5
; GISEL-NEXT: s_setpc_b64 s[30:31]
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
%ret = load volatile <6 x i8>, ptr addrspace(7) %p
diff --git a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
index 444d74166d307..0c414f6b87646 100644
--- a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
@@ -419,15 +419,26 @@ define void @undef_lo3_v4f16(half %arg0) {
}
define void @undef_lo2_v4i16(<2 x i16> %arg0) {
-; GFX8-LABEL: undef_lo2_v4i16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; GFX8-NEXT: ;;#ASMSTART
-; GFX8-NEXT: ; use v[0:1]
-; GFX8-NEXT: ;;#ASMEND
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: undef_lo2_v4i16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-SDAG-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX8-SDAG-NEXT: ;;#ASMSTART
+; GFX8-SDAG-NEXT: ; use v[0:1]
+; GFX8-SDAG-NEXT: ;;#ASMEND
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: undef_lo2_v4i16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-GISEL-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-GISEL-NEXT: ;;#ASMSTART
+; GFX8-GISEL-NEXT: ; use v[0:1]
+; GFX8-GISEL-NEXT: ;;#ASMEND
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: undef_lo2_v4i16:
; GFX9: ; %bb.0:
@@ -462,15 +473,26 @@ define void @undef_lo2_v4i16(<2 x i16> %arg0) {
}
define void @undef_lo2_v4f16(<2 x half> %arg0) {
-; GFX8-LABEL: undef_lo2_v4f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; GFX8-NEXT: ;;#ASMSTART
-; GFX8-NEXT: ; use v[0:1]
-; GFX8-NEXT: ;;#ASMEND
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: undef_lo2_v4f16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-SDAG-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX8-SDAG-NEXT: ;;#ASMSTART
+; GFX8-SDAG-NEXT: ; use v[0:1]
+; GFX8-SDAG-NEXT: ;;#ASMEND
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: undef_lo2_v4f16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-GISEL-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-GISEL-NEXT: ;;#ASMSTART
+; GFX8-GISEL-NEXT: ; use v[0:1]
+; GFX8-GISEL-NEXT: ;;#ASMEND
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: undef_lo2_v4f16:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
index 31a22c067a0fc..ef81554efbc20 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
@@ -2405,28 +2405,13 @@ define <2 x i16> @v_ctlz_zero_poison_v2i16(<2 x i16> %val) {
; GFX9-GISEL-LABEL: v_ctlz_zero_poison_v2i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_pk_lshrrev_b16 v1, 1, v0 op_sel_hi:[0,1]
-; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-GISEL-NEXT: v_pk_lshrrev_b16 v1, 2, v0 op_sel_hi:[0,1]
-; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-GISEL-NEXT: v_pk_lshrrev_b16 v1, 4, v0 op_sel_hi:[0,1]
-; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-GISEL-NEXT: v_pk_lshrrev_b16 v1, 8, v0 op_sel_hi:[0,1]
-; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, -1, v0
-; GFX9-GISEL-NEXT: v_pk_lshrrev_b16 v1, 1, v0 op_sel_hi:[0,1]
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x55555555, v1
-; GFX9-GISEL-NEXT: v_pk_sub_i16 v0, v0, v1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x33333333, v0
-; GFX9-GISEL-NEXT: v_pk_lshrrev_b16 v0, 2, v0 op_sel_hi:[0,1]
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x33333333, v0
-; GFX9-GISEL-NEXT: v_pk_add_u16 v0, v1, v0
-; GFX9-GISEL-NEXT: v_pk_lshrrev_b16 v1, 4, v0 op_sel_hi:[0,1]
-; GFX9-GISEL-NEXT: v_pk_add_u16 v0, v0, v1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xf0f0f0f, v0
-; GFX9-GISEL-NEXT: s_movk_i32 s4, 0x101
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, s4 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%ctlz = call <2 x i16> @llvm.ctlz.v2i16(<2 x i16> %val, i1 true)
ret <2 x i16> %ctlz
@@ -2466,14 +2451,17 @@ define <3 x i16> @v_ctlz_zero_poison_v3i16(<3 x i16> %val) {
; GFX9-GISEL-LABEL: v_ctlz_zero_poison_v3i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, 0x5040100
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
-; GFX9-GISEL-NEXT: v_perm_b32 v0, v2, v0, s4
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%ctlz = call <3 x i16> @llvm.ctlz.v3i16(<3 x i16> %val, i1 true)
ret <3 x i16> %ctlz
@@ -2521,17 +2509,20 @@ define <4 x i16> @v_ctlz_zero_poison_v4i16(<4 x i16> %val) {
; GFX9-GISEL-LABEL: v_ctlz_zero_poison_v4i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v3, v3
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
-; GFX9-GISEL-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-GISEL-NEXT: v_perm_b32 v0, v3, v0, s4
-; GFX9-GISEL-NEXT: v_perm_b32 v1, v2, v1, s4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%ctlz = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %val, i1 true)
ret <4 x i16> %ctlz
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
index 1aaed254c2fc9..365c938d8f6e6 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
@@ -1,16 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck %s -check-prefixes=GFX9,GFX9-SDAG
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck %s -check-prefixes=GFX9,GFX9-GISEL
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck %s -check-prefixes=GFX9,GFX9-GISEL
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck %s -check-prefixes=GFX10,GFX10-SDAG
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck %s -check-prefixes=GFX10,GFX10-GISEL
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10 < %s | FileCheck %s -check-prefixes=GFX10,GFX10-GISEL
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX11-SDAG-TRUE16
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX11-SDAG-FAKE16
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX11-GISEL-TRUE16
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX11-GISEL-FAKE16
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX11-GISEL-FAKE16
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX12-SDAG-TRUE16
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX12-SDAG-FAKE16
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX12-GISEL-TRUE16
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX12-GISEL-FAKE16
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX12-GISEL-FAKE16
declare half @llvm.fma.f16(half, half, half)
declare half @llvm.maxnum.f16(half, half)
@@ -543,34 +543,69 @@ bb:
}
define <2 x i32> @test_D139469_v2f16(<2 x half> %arg) {
-; GFX9-LABEL: test_D139469_v2f16:
-; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x291e
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x211e
-; GFX9-NEXT: v_pk_mul_f16 v1, v0, s4 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_fma_f16 v0, v0, s4, v2 op_sel_hi:[1,0,0]
-; GFX9-NEXT: v_pk_min_f16 v1, v1, v0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: v_cmp_gt_f16_e32 vcc, 0, v1
-; GFX9-NEXT: v_cmp_lt_f16_sdwa s[4:5], v1, v2 src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_D139469_v2f16:
+; GFX9-SDAG: ; %bb.0: ; %bb
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x291e
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x211e
+; GFX9-SDAG-NEXT: v_pk_mul_f16 v1, v0, s4 op_sel_hi:[1,0]
+; GFX9-SDAG-NEXT: v_pk_fma_f16 v0, v0, s4, v2 op_sel_hi:[1,0,0]
+; GFX9-SDAG-NEXT: v_pk_min_f16 v1, v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: v_cmp_gt_f16_e32 vcc, 0, v1
+; GFX9-SDAG-NEXT: v_cmp_lt_f16_sdwa s[4:5], v1, v2 src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_D139469_v2f16:
-; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_movk_i32 s4, 0x211e
-; GFX10-NEXT: v_pk_mul_f16 v1, 0x291e, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_fma_f16 v0, 0x291e, v0, s4 op_sel_hi:[0,1,0]
-; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: v_pk_min_f16 v1, v1, v0
-; GFX10-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v1
-; GFX10-NEXT: v_cmp_lt_f16_sdwa s4, v1, v2 src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_D139469_v2f16:
+; GFX9-GISEL: ; %bb.0: ; %bb
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x291e291e
+; GFX9-GISEL-NEXT: v_pk_mul_f16 v2, v0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-GISEL-NEXT: v_cmp_gt_f16_e32 vcc, 0, v2
+; GFX9-GISEL-NEXT: v_cmp_lt_f16_sdwa s[6:7], v2, v3 src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x211e211e
+; GFX9-GISEL-NEXT: v_pk_fma_f16 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: v_cmp_gt_f16_e64 s[4:5], 0, v0
+; GFX9-GISEL-NEXT: v_cmp_lt_f16_sdwa s[8:9], v0, v3 src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], s[6:7], s[8:9]
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_D139469_v2f16:
+; GFX10-SDAG: ; %bb.0: ; %bb
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_movk_i32 s4, 0x211e
+; GFX10-SDAG-NEXT: v_pk_mul_f16 v1, 0x291e, v0 op_sel_hi:[0,1]
+; GFX10-SDAG-NEXT: v_pk_fma_f16 v0, 0x291e, v0, s4 op_sel_hi:[0,1,0]
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: v_pk_min_f16 v1, v1, v0
+; GFX10-SDAG-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v1
+; GFX10-SDAG-NEXT: v_cmp_lt_f16_sdwa s4, v1, v2 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_D139469_v2f16:
+; GFX10-GISEL: ; %bb.0: ; %bb
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x211e211e
+; GFX10-GISEL-NEXT: v_pk_mul_f16 v2, 0x291e291e, v0
+; GFX10-GISEL-NEXT: v_pk_fma_f16 v0, 0x291e291e, v0, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v2
+; GFX10-GISEL-NEXT: v_cmp_gt_f16_e64 s4, 0, v0
+; GFX10-GISEL-NEXT: v_cmp_lt_f16_sdwa s5, v2, v1 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_cmp_lt_f16_sdwa s6, v0, v1 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s4
+; GFX10-GISEL-NEXT: s_or_b32 s4, s5, s6
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_D139469_v2f16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
@@ -626,17 +661,23 @@ define <2 x i32> @test_D139469_v2f16(<2 x half> %arg) {
; GFX11-GISEL-FAKE16-LABEL: test_D139469_v2f16:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x211e
-; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_f16 v1, 0x291e, v0 op_sel_hi:[0,1]
-; GFX11-GISEL-FAKE16-NEXT: v_pk_fma_f16 v0, 0x291e, v0, s0 op_sel_hi:[0,1,0]
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_pk_min_f16 v0, v1, v0
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x211e211e
+; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_f16 v2, 0x291e291e, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_pk_fma_f16 v0, 0x291e291e, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e64 s0, 0, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e64 s1, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e64 s2, 0, v3
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s0, s1, s2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_D139469_v2f16:
@@ -716,20 +757,24 @@ define <2 x i32> @test_D139469_v2f16(<2 x half> %arg) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x211e
-; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_f16 v1, 0x291e, v0 op_sel_hi:[0,1]
-; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-FAKE16-NEXT: v_pk_fma_f16 v0, 0x291e, v0, s0 op_sel_hi:[0,1,0]
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_pk_min_num_f16 v0, v1, v0
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0
-; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x211e211e
+; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_f16 v2, 0x291e291e, v0
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-FAKE16-NEXT: v_pk_fma_f16 v0, 0x291e291e, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v2
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e64 s0, 0, v0
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e64 s1, 0, v1
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v1
-; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e64 s2, 0, v3
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s0, s1, s2
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
bb:
%i = fmul contract <2 x half> %arg, <half 0xH291E, half 0xH291E>
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
index 6f07c945e68a2..98e24aa3340a3 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-TRUE16 %s
@@ -496,17 +496,29 @@ define amdgpu_ps half @test_fmaximum_f16_ss(half inreg %a, half inreg %b) {
}
define amdgpu_ps <2 x half> @test_fmaximum_v2f16_vv(<2 x half> %a, <2 x half> %b) {
-; GFX9-LABEL: test_fmaximum_v2f16_vv:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_max_f16 v2, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v0, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v2, vcc
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: v_perm_b32 v0, v0, v4, s0
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fmaximum_v2f16_vv:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_pk_max_f16 v2, v0, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, v0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v3, v2, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v0, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_mov_b32 s0, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v0, v4, s0
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fmaximum_v2f16_vv:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_pk_max_f16 v2, v0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7e00
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v0, v1
+; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v3, v2, s[0:1]
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fmaximum_v2f16_vv:
; GFX1170: ; %bb.0:
@@ -522,22 +534,45 @@ define amdgpu_ps <2 x half> @test_fmaximum_v2f16_vv(<2 x half> %a, <2 x half> %b
}
define amdgpu_ps <2 x half> @test_fmaximum_v2f16_ss(<2 x half> inreg %a, <2 x half> inreg %b) {
-; GFX9-LABEL: test_fmaximum_v2f16_ss:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: v_pk_max_f16 v1, s0, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v3
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fmaximum_v2f16_ss:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-SDAG-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-SDAG-NEXT: v_pk_max_f16 v1, s0, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-SDAG-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v3
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fmaximum_v2f16_ss:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-GISEL-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-GISEL-NEXT: v_pk_max_f16 v1, s0, v0
+; GFX9-GISEL-NEXT: s_lshr_b32 s3, s0, 16
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s3, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX9-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, s2, 0x7e00
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s1, s3, 0x7e00
+; GFX9-GISEL-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fmaximum_v2f16_ss:
; GFX1170: ; %bb.0:
@@ -553,20 +588,38 @@ define amdgpu_ps <2 x half> @test_fmaximum_v2f16_ss(<2 x half> inreg %a, <2 x ha
}
define amdgpu_ps <3 x half> @test_fmaximum_v3f16_vv(<3 x half> %a, <3 x half> %b) {
-; GFX9-LABEL: test_fmaximum_v3f16_vv:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_max_f16 v4, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc
-; GFX9-NEXT: v_pk_max_f16 v3, v0, v2
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v3, vcc
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v5, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: v_perm_b32 v0, v0, v4, s0
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fmaximum_v3f16_vv:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_pk_max_f16 v4, v1, v3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc
+; GFX9-SDAG-NEXT: v_pk_max_f16 v3, v0, v2
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v5, v3, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v0, v5, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_mov_b32 s0, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v0, v4, s0
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fmaximum_v3f16_vv:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_pk_max_f16 v4, v0, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0x7e00
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v6, v5, vcc
+; GFX9-GISEL-NEXT: v_pk_max_f16 v2, v1, v3
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-SDAG-LABEL: test_fmaximum_v3f16_vv:
; GFX1170-SDAG: ; %bb.0:
@@ -614,27 +667,57 @@ define amdgpu_ps <3 x half> @test_fmaximum_v3f16_vv(<3 x half> %a, <3 x half> %b
}
define amdgpu_ps <3 x half> @test_fmaximum_v3f16_ss(<3 x half> inreg %a, <3 x half> inreg %b) {
-; GFX9-LABEL: test_fmaximum_v3f16_ss:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_pk_max_f16 v1, s1, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s2
-; GFX9-NEXT: s_lshr_b32 s1, s2, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_pk_max_f16 v3, s0, v3
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v4
-; GFX9-NEXT: v_cndmask_b32_sdwa v2, v2, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fmaximum_v3f16_ss:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: v_pk_max_f16 v1, s1, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-SDAG-NEXT: s_lshr_b32 s1, s2, 16
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-SDAG-NEXT: v_pk_max_f16 v3, s0, v3
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-SDAG-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v4
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fmaximum_v3f16_ss:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_lshr_b32 s5, s2, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-GISEL-NEXT: v_pk_max_f16 v1, s0, v0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s4, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-GISEL-NEXT: s_lshr_b32 s5, s2, 16
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, s2, 0x7e00
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX9-GISEL-NEXT: v_pk_max_f16 v1, s1, v0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
+; GFX9-GISEL-NEXT: s_cselect_b32 s2, s5, 0x7e00
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s1, s3, 0x7e00
+; GFX9-GISEL-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-SDAG-LABEL: test_fmaximum_v3f16_ss:
; GFX1170-SDAG: ; %bb.0:
@@ -681,23 +764,43 @@ define amdgpu_ps <3 x half> @test_fmaximum_v3f16_ss(<3 x half> inreg %a, <3 x ha
}
define amdgpu_ps <4 x half> @test_fmaximum_v4f16(<4 x half> %a, <4 x half> %b) {
-; GFX9-LABEL: test_fmaximum_v4f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_max_f16 v4, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v5, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_pk_max_f16 v3, v0, v2
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v3, vcc
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v5, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: v_perm_b32 v0, v0, v4, s0
-; GFX9-NEXT: v_perm_b32 v1, v1, v6, s0
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fmaximum_v4f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_pk_max_f16 v4, v1, v3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v5, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_pk_max_f16 v3, v0, v2
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v5, v3, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v0, v5, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_mov_b32 s0, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v0, v4, s0
+; GFX9-SDAG-NEXT: v_perm_b32 v1, v1, v6, s0
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fmaximum_v4f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_pk_max_f16 v4, v0, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0x7e00
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v6, v5, vcc
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v4
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-GISEL-NEXT: v_pk_max_f16 v2, v1, v3
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v1, v3
+; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, v6, v2, s[0:1]
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v2, v6, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fmaximum_v4f16:
; GFX1170: ; %bb.0:
@@ -715,35 +818,77 @@ define amdgpu_ps <4 x half> @test_fmaximum_v4f16(<4 x half> %a, <4 x half> %b) {
}
define amdgpu_ps <4 x half> @test_fmaximum_v4f16_ss(<4 x half> inreg %a, <4 x half> inreg %b) {
-; GFX9-LABEL: test_fmaximum_v4f16_ss:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: s_lshr_b32 s3, s3, 16
-; GFX9-NEXT: v_pk_max_f16 v1, s1, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v1, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v4, s2
-; GFX9-NEXT: s_lshr_b32 s1, s2, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_pk_max_f16 v4, s0, v4
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v5
-; GFX9-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fmaximum_v4f16_ss:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: s_lshr_b32 s3, s3, 16
+; GFX9-SDAG-NEXT: v_pk_max_f16 v1, s1, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
+; GFX9-SDAG-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v3, v2, v1, vcc
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, s2
+; GFX9-SDAG-NEXT: s_lshr_b32 s1, s2, 16
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-SDAG-NEXT: v_pk_max_f16 v4, s0, v4
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-SDAG-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v5
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-SDAG-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fmaximum_v4f16_ss:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-GISEL-NEXT: v_pk_max_f16 v1, s0, v0
+; GFX9-GISEL-NEXT: s_lshr_b32 s5, s0, 16
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s5, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; GFX9-GISEL-NEXT: s_lshr_b32 s5, s4, 16
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, s4, 0x7e00
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s2, s5, 0x7e00
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT: s_lshr_b32 s3, s3, 16
+; GFX9-GISEL-NEXT: v_pk_max_f16 v1, s1, v0
+; GFX9-GISEL-NEXT: s_lshr_b32 s4, s1, 16
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s4, v0
+; GFX9-GISEL-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX9-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s3, 1, 0
+; GFX9-GISEL-NEXT: s_lshr_b32 s4, s2, 16
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s1, s2, 0x7e00
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s3, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s2, s4, 0x7e00
+; GFX9-GISEL-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fmaximum_v4f16_ss:
; GFX1170: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
index 1a7707f0d261a..8ba95ee3550c0 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.70 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.70 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-TRUE16 %s
@@ -496,17 +496,29 @@ define amdgpu_ps half @test_fminimum_f16_ss(half inreg %a, half inreg %b) {
}
define amdgpu_ps <2 x half> @test_fminimum_v2f16_vv(<2 x half> %a, <2 x half> %b) {
-; GFX9-LABEL: test_fminimum_v2f16_vv:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_min_f16 v2, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v0, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v2, vcc
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: v_perm_b32 v0, v0, v4, s0
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fminimum_v2f16_vv:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_pk_min_f16 v2, v0, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, v0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v3, v2, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v0, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_mov_b32 s0, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v0, v4, s0
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fminimum_v2f16_vv:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_pk_min_f16 v2, v0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7e00
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v0, v1
+; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v3, v2, s[0:1]
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fminimum_v2f16_vv:
; GFX1170: ; %bb.0:
@@ -522,22 +534,45 @@ define amdgpu_ps <2 x half> @test_fminimum_v2f16_vv(<2 x half> %a, <2 x half> %b
}
define amdgpu_ps <2 x half> @test_fminimum_v2f16_ss(<2 x half> inreg %a, <2 x half> inreg %b) {
-; GFX9-LABEL: test_fminimum_v2f16_ss:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: v_pk_min_f16 v1, s0, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v3
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fminimum_v2f16_ss:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-SDAG-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-SDAG-NEXT: v_pk_min_f16 v1, s0, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-SDAG-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v3
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fminimum_v2f16_ss:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-GISEL-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-GISEL-NEXT: v_pk_min_f16 v1, s0, v0
+; GFX9-GISEL-NEXT: s_lshr_b32 s3, s0, 16
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s3, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX9-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, s2, 0x7e00
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s1, s3, 0x7e00
+; GFX9-GISEL-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fminimum_v2f16_ss:
; GFX1170: ; %bb.0:
@@ -553,20 +588,38 @@ define amdgpu_ps <2 x half> @test_fminimum_v2f16_ss(<2 x half> inreg %a, <2 x ha
}
define amdgpu_ps <3 x half> @test_fminimum_v3f16_vv(<3 x half> %a, <3 x half> %b) {
-; GFX9-LABEL: test_fminimum_v3f16_vv:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_min_f16 v4, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc
-; GFX9-NEXT: v_pk_min_f16 v3, v0, v2
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v3, vcc
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v5, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: v_perm_b32 v0, v0, v4, s0
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fminimum_v3f16_vv:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_pk_min_f16 v4, v1, v3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc
+; GFX9-SDAG-NEXT: v_pk_min_f16 v3, v0, v2
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v5, v3, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v0, v5, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_mov_b32 s0, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v0, v4, s0
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fminimum_v3f16_vv:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_pk_min_f16 v4, v0, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0x7e00
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v6, v5, vcc
+; GFX9-GISEL-NEXT: v_pk_min_f16 v2, v1, v3
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-SDAG-LABEL: test_fminimum_v3f16_vv:
; GFX1170-SDAG: ; %bb.0:
@@ -614,27 +667,57 @@ define amdgpu_ps <3 x half> @test_fminimum_v3f16_vv(<3 x half> %a, <3 x half> %b
}
define amdgpu_ps <3 x half> @test_fminimum_v3f16_ss(<3 x half> inreg %a, <3 x half> inreg %b) {
-; GFX9-LABEL: test_fminimum_v3f16_ss:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_pk_min_f16 v1, s1, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s2
-; GFX9-NEXT: s_lshr_b32 s1, s2, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_pk_min_f16 v3, s0, v3
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v4
-; GFX9-NEXT: v_cndmask_b32_sdwa v2, v2, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fminimum_v3f16_ss:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: v_pk_min_f16 v1, s1, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-SDAG-NEXT: s_lshr_b32 s1, s2, 16
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-SDAG-NEXT: v_pk_min_f16 v3, s0, v3
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-SDAG-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v4
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fminimum_v3f16_ss:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_lshr_b32 s5, s2, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-GISEL-NEXT: v_pk_min_f16 v1, s0, v0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s4, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-GISEL-NEXT: s_lshr_b32 s5, s2, 16
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, s2, 0x7e00
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX9-GISEL-NEXT: v_pk_min_f16 v1, s1, v0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
+; GFX9-GISEL-NEXT: s_cselect_b32 s2, s5, 0x7e00
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s1, s3, 0x7e00
+; GFX9-GISEL-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-SDAG-LABEL: test_fminimum_v3f16_ss:
; GFX1170-SDAG: ; %bb.0:
@@ -681,23 +764,43 @@ define amdgpu_ps <3 x half> @test_fminimum_v3f16_ss(<3 x half> inreg %a, <3 x ha
}
define amdgpu_ps <4 x half> @test_fminimum_v4f16(<4 x half> %a, <4 x half> %b) {
-; GFX9-LABEL: test_fminimum_v4f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_min_f16 v4, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v5, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_pk_min_f16 v3, v0, v2
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v3, vcc
-; GFX9-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v5, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: v_perm_b32 v0, v0, v4, s0
-; GFX9-NEXT: v_perm_b32 v1, v1, v6, s0
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fminimum_v4f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_pk_min_f16 v4, v1, v3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v5, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_pk_min_f16 v3, v0, v2
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v5, v3, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v0, v5, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_mov_b32 s0, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v0, v4, s0
+; GFX9-SDAG-NEXT: v_perm_b32 v1, v1, v6, s0
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fminimum_v4f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_pk_min_f16 v4, v0, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0x7e00
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, v0, v2
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v6, v5, vcc
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v4
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-GISEL-NEXT: v_pk_min_f16 v2, v1, v3
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v1, v3
+; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, v6, v2, s[0:1]
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v2, v6, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fminimum_v4f16:
; GFX1170: ; %bb.0:
@@ -715,35 +818,77 @@ define amdgpu_ps <4 x half> @test_fminimum_v4f16(<4 x half> %a, <4 x half> %b) {
}
define amdgpu_ps <4 x half> @test_fminimum_v4f16_ss(<4 x half> inreg %a, <4 x half> inreg %b) {
-; GFX9-LABEL: test_fminimum_v4f16_ss:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: s_lshr_b32 s3, s3, 16
-; GFX9-NEXT: v_pk_min_f16 v1, s1, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7e00
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v1, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v4, s2
-; GFX9-NEXT: s_lshr_b32 s1, s2, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_pk_min_f16 v4, s0, v4
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
-; GFX9-NEXT: v_cmp_o_f16_e32 vcc, s0, v5
-; GFX9-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
-; GFX9-NEXT: ; return to shader part epilog
+; GFX9-SDAG-LABEL: test_fminimum_v4f16_ss:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: s_lshr_b32 s3, s3, 16
+; GFX9-SDAG-NEXT: v_pk_min_f16 v1, s1, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7e00
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
+; GFX9-SDAG-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v3, v2, v1, vcc
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, s2
+; GFX9-SDAG-NEXT: s_lshr_b32 s1, s2, 16
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-SDAG-NEXT: v_pk_min_f16 v4, s0, v4
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-SDAG-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v5
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-SDAG-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX9-GISEL-LABEL: test_fminimum_v4f16_ss:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-GISEL-NEXT: v_pk_min_f16 v1, s0, v0
+; GFX9-GISEL-NEXT: s_lshr_b32 s5, s0, 16
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s0, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s5, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; GFX9-GISEL-NEXT: s_lshr_b32 s5, s4, 16
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s0, s4, 0x7e00
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s2, s5, 0x7e00
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT: s_lshr_b32 s3, s3, 16
+; GFX9-GISEL-NEXT: v_pk_min_f16 v1, s1, v0
+; GFX9-GISEL-NEXT: s_lshr_b32 s4, s1, 16
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, s4, v0
+; GFX9-GISEL-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX9-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX9-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s3, 1, 0
+; GFX9-GISEL-NEXT: s_lshr_b32 s4, s2, 16
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s1, s2, 0x7e00
+; GFX9-GISEL-NEXT: s_cmp_lg_u32 s3, 0
+; GFX9-GISEL-NEXT: s_cselect_b32 s2, s4, 0x7e00
+; GFX9-GISEL-NEXT: s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fminimum_v4f16_ss:
; GFX1170: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
index 153b1ffbeca46..2ce2efcfe49ec 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
@@ -6594,30 +6594,55 @@ define half @v_contract_mul_add_f16_select_4_128(i32 %arg, half %x, half %y) {
}
define <2 x half> @v_mul_v2f16_select_64_1(<2 x i32> %arg, <2 x half> %x) {
-; GFX9-LABEL: v_mul_v2f16_select_64_1:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, 0x3c00
-; GFX9-NEXT: v_mov_b32_e32 v4, 0x5400
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: v_pk_mul_f16 v0, v2, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_mul_v2f16_select_64_1:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x3c00
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0x5400
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: v_pk_mul_f16 v0, v2, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_mul_v2f16_select_64_1:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, 0x5400
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x3c00, v3, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x3c00, v3, vcc_lo
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: v_pk_mul_f16 v0, v2, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_mul_v2f16_select_64_1:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_mul_v2f16_select_64_1:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0x5400
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0x3c00, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0x3c00, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_pk_mul_f16 v0, v2, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_mul_v2f16_select_64_1:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc_lo
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_64_1:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -6656,13 +6681,15 @@ define <2 x half> @v_mul_v2f16_select_64_1(<2 x i32> %arg, <2 x half> %x) {
; GFX11-GISEL-FAKE16-LABEL: v_mul_v2f16_select_64_1:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x5400
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x3c00, v3, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x3c00, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v3, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq <2 x i32> %arg, zeroinitializer
%select.pow2 = select <2 x i1> %cond, <2 x half> <half 64.0, half 64.0>, <2 x half> <half 1.0, half 1.0>
@@ -6671,30 +6698,55 @@ define <2 x half> @v_mul_v2f16_select_64_1(<2 x i32> %arg, <2 x half> %x) {
}
define <2 x half> @v_mul_v2f16_select_1_64(<2 x i32> %arg, <2 x half> %x) {
-; GFX9-LABEL: v_mul_v2f16_select_1_64:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, 0x5400
-; GFX9-NEXT: v_mov_b32_e32 v4, 0x3c00
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: v_pk_mul_f16 v0, v2, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_mul_v2f16_select_1_64:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x5400
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0x3c00
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: v_pk_mul_f16 v0, v2, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_mul_v2f16_select_1_64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, 0x3c00
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x5400, v3, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x5400, v3, vcc_lo
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: v_pk_mul_f16 v0, v2, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_mul_v2f16_select_1_64:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_mul_v2f16_select_1_64:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0x3c00
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0x5400, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0x5400, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_pk_mul_f16 v0, v2, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_mul_v2f16_select_1_64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_1_64:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -6733,13 +6785,15 @@ define <2 x half> @v_mul_v2f16_select_1_64(<2 x i32> %arg, <2 x half> %x) {
; GFX11-GISEL-FAKE16-LABEL: v_mul_v2f16_select_1_64:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x3c00
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x5400, v3, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x5400, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v3, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq <2 x i32> %arg, zeroinitializer
%select.pow2 = select <2 x i1> %cond, <2 x half> <half 1.0, half 1.0>, <2 x half> <half 64.0, half 64.0>
@@ -6748,30 +6802,57 @@ define <2 x half> @v_mul_v2f16_select_1_64(<2 x i32> %arg, <2 x half> %x) {
}
define <2 x half> @v_mul_v2f16_select_n1_n64(<2 x i32> %arg, <2 x half> %x) {
-; GFX9-LABEL: v_mul_v2f16_select_n1_n64:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xd400
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xbc00
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: v_pk_mul_f16 v0, v2, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_mul_v2f16_select_n1_n64:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0xd400
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0xbc00
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: v_pk_mul_f16 v0, v2, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_mul_v2f16_select_n1_n64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, 0xbc00
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xd400, v3, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0xd400, v3, vcc_lo
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: v_pk_mul_f16 v0, v2, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_mul_v2f16_select_n1_n64:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_mul_v2f16_select_n1_n64:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0xbc00
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0xd400, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0xd400, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_pk_mul_f16 v0, v2, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_mul_v2f16_select_n1_n64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_n1_n64:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -6811,13 +6892,16 @@ define <2 x half> @v_mul_v2f16_select_n1_n64(<2 x i32> %arg, <2 x half> %x) {
; GFX11-GISEL-FAKE16-LABEL: v_mul_v2f16_select_n1_n64:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0xbc00
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0xd400, v3, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0xd400, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v3, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq <2 x i32> %arg, zeroinitializer
%select.pow2 = select <2 x i1> %cond, <2 x half> <half -1.0, half -1.0>, <2 x half> <half -64.0, half -64.0>
@@ -6980,30 +7064,57 @@ define <2 x half> @v_mul_v2f16_select_n128_n64(<2 x i32> %arg, <2 x half> %x) {
}
define <2 x half> @v_mul_v2f16_select_n128_n16(<2 x i32> %arg, <2 x half> %x) {
-; GFX9-LABEL: v_mul_v2f16_select_n128_n16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xcc00
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xd800
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: v_pk_mul_f16 v0, v2, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_mul_v2f16_select_n128_n16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0xcc00
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0xd800
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: v_pk_mul_f16 v0, v2, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_mul_v2f16_select_n128_n16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, 0xd800
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xcc00, v3, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0xcc00, v3, vcc_lo
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: v_pk_mul_f16 v0, v2, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_mul_v2f16_select_n128_n16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 4, 7, vcc
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 4, 7, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_mul_v2f16_select_n128_n16:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0xd800
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0xcc00, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0xcc00, v3, vcc_lo
+; GFX10-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_pk_mul_f16 v0, v2, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_mul_v2f16_select_n128_n16:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 4, 7, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 4, 7, vcc_lo
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_n128_n16:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -7043,13 +7154,16 @@ define <2 x half> @v_mul_v2f16_select_n128_n16(<2 x i32> %arg, <2 x half> %x) {
; GFX11-GISEL-FAKE16-LABEL: v_mul_v2f16_select_n128_n16:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0xd800
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0xcc00, v3, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0xcc00, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 4, 7, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 4, 7, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v3, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq <2 x i32> %arg, zeroinitializer
%select.pow2 = select <2 x i1> %cond, <2 x half> <half -128.0, half -128.0>, <2 x half> <half -16.0, half -16.0>
@@ -7058,30 +7172,57 @@ define <2 x half> @v_mul_v2f16_select_n128_n16(<2 x i32> %arg, <2 x half> %x) {
}
define <2 x half> @v_contract_mul_add_v2f16_select_64_1(<2 x i32> %arg, <2 x half> %x, <2 x half> %y) {
-; GFX9-LABEL: v_contract_mul_add_v2f16_select_64_1:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, 0x3c00
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x5400
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: v_pk_fma_f16 v0, v2, v0, v3
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_contract_mul_add_v2f16_select_64_1:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0x3c00
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, 0x5400
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_contract_mul_add_v2f16_select_64_1:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x5400
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x3c00, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x3c00, v4, vcc_lo
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: v_pk_fma_f16 v0, v2, v0, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_contract_mul_add_v2f16_select_64_1:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_contract_mul_add_v2f16_select_64_1:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0x5400
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0x3c00, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0x3c00, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_contract_mul_add_v2f16_select_64_1:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc_lo
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_v2f16_select_64_1:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -7121,13 +7262,16 @@ define <2 x half> @v_contract_mul_add_v2f16_select_64_1(<2 x i32> %arg, <2 x hal
; GFX11-GISEL-FAKE16-LABEL: v_contract_mul_add_v2f16_select_64_1:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, 0x5400
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x3c00, v4, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x3c00, v4, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v4, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_pk_add_f16 v0, v0, v3
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq <2 x i32> %arg, zeroinitializer
%select.pow2 = select <2 x i1> %cond, <2 x half> <half 64.0, half 64.0>, <2 x half> <half 1.0, half 1.0>
@@ -7137,30 +7281,57 @@ define <2 x half> @v_contract_mul_add_v2f16_select_64_1(<2 x i32> %arg, <2 x hal
}
define <2 x half> @v_contract_mul_add_v2f16_select_1_64(<2 x i32> %arg, <2 x half> %x, <2 x half> %y) {
-; GFX9-LABEL: v_contract_mul_add_v2f16_select_1_64:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, 0x5400
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x3c00
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: v_pk_fma_f16 v0, v2, v0, v3
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_contract_mul_add_v2f16_select_1_64:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0x5400
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, 0x3c00
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_contract_mul_add_v2f16_select_1_64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x3c00
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x5400, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x5400, v4, vcc_lo
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: v_pk_fma_f16 v0, v2, v0, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_contract_mul_add_v2f16_select_1_64:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_contract_mul_add_v2f16_select_1_64:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0x3c00
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0x5400, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0x5400, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_contract_mul_add_v2f16_select_1_64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_v2f16_select_1_64:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -7200,13 +7371,16 @@ define <2 x half> @v_contract_mul_add_v2f16_select_1_64(<2 x i32> %arg, <2 x hal
; GFX11-GISEL-FAKE16-LABEL: v_contract_mul_add_v2f16_select_1_64:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, 0x3c00
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x5400, v4, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x5400, v4, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v4, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_pk_add_f16 v0, v0, v3
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq <2 x i32> %arg, zeroinitializer
%select.pow2 = select <2 x i1> %cond, <2 x half> <half 1.0, half 1.0>, <2 x half> <half 64.0, half 64.0>
@@ -7216,30 +7390,59 @@ define <2 x half> @v_contract_mul_add_v2f16_select_1_64(<2 x i32> %arg, <2 x hal
}
define <2 x half> @v_contract_mul_add_v2f16_select_n64_n1(<2 x i32> %arg, <2 x half> %x, <2 x half> %y) {
-; GFX9-LABEL: v_contract_mul_add_v2f16_select_n64_n1:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xbc00
-; GFX9-NEXT: v_mov_b32_e32 v5, 0xd400
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: v_pk_fma_f16 v0, v2, v0, v3
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_contract_mul_add_v2f16_select_n64_n1:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0xbc00
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, 0xd400
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_contract_mul_add_v2f16_select_n64_n1:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xd400
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xbc00, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0xbc00, v4, vcc_lo
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: v_pk_fma_f16 v0, v2, v0, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_contract_mul_add_v2f16_select_n64_n1:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_contract_mul_add_v2f16_select_n64_n1:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0xd400
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0xbc00, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0xbc00, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_contract_mul_add_v2f16_select_n64_n1:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc_lo
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_v2f16_select_n64_n1:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -7280,13 +7483,17 @@ define <2 x half> @v_contract_mul_add_v2f16_select_n64_n1(<2 x i32> %arg, <2 x h
; GFX11-GISEL-FAKE16-LABEL: v_contract_mul_add_v2f16_select_n64_n1:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, 0xd400
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0xbc00, v4, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0xbc00, v4, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX11-GISEL-FAKE16-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v4, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_pk_add_f16 v0, v0, v3
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq <2 x i32> %arg, zeroinitializer
%select.pow2 = select <2 x i1> %cond, <2 x half> <half -64.0, half -64.0>, <2 x half> <half -1.0, half -1.0>
@@ -7296,30 +7503,59 @@ define <2 x half> @v_contract_mul_add_v2f16_select_n64_n1(<2 x i32> %arg, <2 x h
}
define <2 x half> @v_contract_mul_add_v2f16_select_n1_n64(<2 x i32> %arg, <2 x half> %x, <2 x half> %y) {
-; GFX9-LABEL: v_contract_mul_add_v2f16_select_n1_n64:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xd400
-; GFX9-NEXT: v_mov_b32_e32 v5, 0xbc00
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: v_pk_fma_f16 v0, v2, v0, v3
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_contract_mul_add_v2f16_select_n1_n64:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0xd400
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, 0xbc00
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_contract_mul_add_v2f16_select_n1_n64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xbc00
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xd400, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0xd400, v4, vcc_lo
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: v_pk_fma_f16 v0, v2, v0, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_contract_mul_add_v2f16_select_n1_n64:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_contract_mul_add_v2f16_select_n1_n64:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0xbc00
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0xd400, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0xd400, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_contract_mul_add_v2f16_select_n1_n64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_v2f16_select_n1_n64:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -7360,13 +7596,17 @@ define <2 x half> @v_contract_mul_add_v2f16_select_n1_n64(<2 x i32> %arg, <2 x h
; GFX11-GISEL-FAKE16-LABEL: v_contract_mul_add_v2f16_select_n1_n64:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, 0xbc00
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0xd400, v4, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0xd400, v4, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX11-GISEL-FAKE16-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v4, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_pk_add_f16 v0, v0, v3
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq <2 x i32> %arg, zeroinitializer
%select.pow2 = select <2 x i1> %cond, <2 x half> <half -1.0, half -1.0>, <2 x half> <half -64.0, half -64.0>
@@ -7454,30 +7694,57 @@ define <2 x half> @v_contract_mul_add_v2f16_select_128_64(<2 x i32> %arg, <2 x h
}
define <2 x half> @v_contract_mul_add_v2f16_select_128_4(<2 x i32> %arg, <2 x half> %x, <2 x half> %y) {
-; GFX9-LABEL: v_contract_mul_add_v2f16_select_128_4:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, 0x4400
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x5800
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: v_pk_fma_f16 v0, v2, v0, v3
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_contract_mul_add_v2f16_select_128_4:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0x4400
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, 0x5800
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_contract_mul_add_v2f16_select_128_4:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x5800
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x4400, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x4400, v4, vcc_lo
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: v_pk_fma_f16 v0, v2, v0, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_contract_mul_add_v2f16_select_128_4:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 2, 7, vcc
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 2, 7, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_contract_mul_add_v2f16_select_128_4:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0x5800
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0x4400, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0x4400, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_contract_mul_add_v2f16_select_128_4:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 2, 7, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 2, 7, vcc_lo
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_v2f16_select_128_4:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -7517,13 +7784,16 @@ define <2 x half> @v_contract_mul_add_v2f16_select_128_4(<2 x i32> %arg, <2 x ha
; GFX11-GISEL-FAKE16-LABEL: v_contract_mul_add_v2f16_select_128_4:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, 0x5800
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x4400, v4, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x4400, v4, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 2, 7, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 2, 7, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v4, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_pk_add_f16 v0, v0, v3
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq <2 x i32> %arg, zeroinitializer
%select.pow2 = select <2 x i1> %cond, <2 x half> <half 128.0, half 128.0>, <2 x half> <half 4.0, half 4.0>
@@ -7611,30 +7881,57 @@ define <2 x half> @v_contract_mul_add_v2f16_select_2_4(<2 x i32> %arg, <2 x half
}
define <2 x half> @v_contract_mul_add_v2f16_select_4_128(<2 x i32> %arg, <2 x half> %x, <2 x half> %y) {
-; GFX9-LABEL: v_contract_mul_add_v2f16_select_4_128:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, 0x5800
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x4400
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: v_pk_fma_f16 v0, v2, v0, v3
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: v_contract_mul_add_v2f16_select_4_128:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0x5800
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, 0x4400
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_contract_mul_add_v2f16_select_4_128:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x4400
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x5800, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x5800, v4, vcc_lo
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX10-NEXT: v_pk_fma_f16 v0, v2, v0, v3
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: v_contract_mul_add_v2f16_select_4_128:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 7, 2, vcc
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 7, 2, vcc
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: v_contract_mul_add_v2f16_select_4_128:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0x4400
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0x5800, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, 0x5800, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_contract_mul_add_v2f16_select_4_128:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 7, 2, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 7, 2, vcc_lo
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_v2f16_select_4_128:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -7674,13 +7971,16 @@ define <2 x half> @v_contract_mul_add_v2f16_select_4_128(<2 x i32> %arg, <2 x ha
; GFX11-GISEL-FAKE16-LABEL: v_contract_mul_add_v2f16_select_4_128:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, 0x4400
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x5800, v4, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x5800, v4, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_fma_f16 v0, v2, v0, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 7, 2, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 7, 2, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v4, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_pk_add_f16 v0, v0, v3
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cond = icmp eq <2 x i32> %arg, zeroinitializer
%select.pow2 = select <2 x i1> %cond, <2 x half> <half 4.0, half 4.0>, <2 x half> <half 128.0, half 128.0>
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
index e1a5e2438d06a..6f0d53787f1f9 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
@@ -88,15 +88,24 @@ define half @fptrunc_v2f32_v2f16_extract_uses(<2 x float> %src) {
}
define half @fptrunc_v3f32_v3f16_extract_uses(<3 x float> %vec_float) {
-; GFX950-LABEL: fptrunc_v3f32_v3f16_extract_uses:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX950-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX950-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX950-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX950-NEXT: v_add_f16_e32 v0, v2, v0
-; GFX950-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: fptrunc_v3f32_v3f16_extract_uses:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX950-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX950-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX950-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT: v_add_f16_e32 v0, v2, v0
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: fptrunc_v3f32_v3f16_extract_uses:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_e32 v0, v2, v0
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%vec_half = fptrunc <3 x float> %vec_float to <3 x half>
%f0 = extractelement <3 x half> %vec_half, i64 0
%f1 = extractelement <3 x half> %vec_half, i64 1
diff --git a/llvm/test/CodeGen/AMDGPU/freeze.ll b/llvm/test/CodeGen/AMDGPU/freeze.ll
index 69f18ecbf89ac..ad63b129cbe7a 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze.ll
@@ -1,24 +1,24 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu6.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX6,GFX6-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu6.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX6,GFX6-GISEL %s
+; xUN: llc -global-isel=1 -mtriple=amdgpu6.00-mesa-mesa3d -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX6,GFX6-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; xUN: llc -global-isel=1 -mtriple=amdgpu7.00-mesa-mesa3d -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG,GFX8-GFX803-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; xUN: llc -global-isel=1 -mtriple=amdgpu8.03-mesa-mesa3d -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX9,GFX8-SDAG,GFX8-GFX900-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; xUN: llc -global-isel=1 -mtriple=amdgpu9.00-mesa-mesa3d -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; xUN: llc -global-isel=1 -mtriple=amdgpu10.10-mesa-mesa3d -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; FIXME-TRUE16 enable gisel
; XUN: llc -global-isel=1 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
+; xUN: llc -global-isel=1 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
;FIXME: add regbankcombine for
; %val = ...
@@ -6786,45 +6786,91 @@ define void @freeze_v2i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
}
define void @freeze_v3i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
-; GFX6-LABEL: freeze_v3i16:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
-; GFX6-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX6-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: freeze_v3i16:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: freeze_v3i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
-; GFX7-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX6-GISEL-LABEL: freeze_v3i16:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: buffer_store_short v4, v[2:3], s[4:7], 0 addr64 offset:2
+; GFX6-GISEL-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: freeze_v3i16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v2
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: flat_store_short v[4:5], v1
-; GFX8-NEXT: flat_store_dword v[2:3], v0
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: freeze_v3i16:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
+; GFX7-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: freeze_v3i16:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX7-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-GISEL-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: buffer_store_short v4, v[2:3], s[4:7], 0 addr64 offset:2
+; GFX7-GISEL-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GFX803-SDAG-LABEL: freeze_v3i16:
+; GFX8-GFX803-SDAG: ; %bb.0:
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 4, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_store_short v[4:5], v1
+; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[2:3], v0
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: freeze_v3i16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-GISEL-NEXT: v_add_u32_e32 v4, vcc, 2, v2
+; GFX8-GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GFX8-GISEL-NEXT: v_add_u32_e32 v6, vcc, 4, v2
+; GFX8-GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX8-GISEL-NEXT: flat_store_short v[2:3], v0
+; GFX8-GISEL-NEXT: flat_store_short v[4:5], v8
+; GFX8-GISEL-NEXT: flat_store_short v[6:7], v1
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GFX900-SDAG-LABEL: freeze_v3i16:
; GFX8-GFX900-SDAG: ; %bb.0:
@@ -7445,63 +7491,109 @@ define void @freeze_v2f16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
}
define void @freeze_v3f16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
-; GFX6-LABEL: freeze_v3f16:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX6-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
-; GFX6-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX6-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: freeze_v3f16:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX6-SDAG-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: freeze_v3f16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
-; GFX7-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX6-GISEL-LABEL: freeze_v3f16:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: buffer_store_short v4, v[2:3], s[4:7], 0 addr64 offset:2
+; GFX6-GISEL-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: freeze_v3f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v2
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: flat_store_short v[4:5], v1
-; GFX8-NEXT: flat_store_dword v[2:3], v0
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: freeze_v3f16:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
+; GFX7-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: freeze_v3f16:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX7-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-GISEL-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: buffer_store_short v4, v[2:3], s[4:7], 0 addr64 offset:2
+; GFX7-GISEL-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GFX803-SDAG-LABEL: freeze_v3f16:
+; GFX8-GFX803-SDAG: ; %bb.0:
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 4, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_store_short v[4:5], v1
+; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[2:3], v0
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: freeze_v3f16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-GISEL-NEXT: v_add_u32_e32 v4, vcc, 2, v2
+; GFX8-GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GFX8-GISEL-NEXT: v_add_u32_e32 v6, vcc, 4, v2
+; GFX8-GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX8-GISEL-NEXT: flat_store_short v[2:3], v0
+; GFX8-GISEL-NEXT: flat_store_short v[4:5], v8
+; GFX8-GISEL-NEXT: flat_store_short v[6:7], v1
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GFX900-SDAG-LABEL: freeze_v3f16:
; GFX8-GFX900-SDAG: ; %bb.0:
@@ -8124,63 +8216,109 @@ define void @freeze_v2bf16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
}
define void @freeze_v3bf16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
-; GFX6-LABEL: freeze_v3bf16:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_alignbit_b32 v0, v4, v0, 16
-; GFX6-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
-; GFX6-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX6-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: freeze_v3bf16:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-SDAG-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX6-SDAG-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX6-SDAG-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-SDAG-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX6-SDAG-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: freeze_v3bf16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_alignbit_b32 v0, v4, v0, 16
-; GFX7-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
-; GFX7-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX6-GISEL-LABEL: freeze_v3bf16:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: buffer_store_short v4, v[2:3], s[4:7], 0 addr64 offset:2
+; GFX6-GISEL-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: freeze_v3bf16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v2
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: flat_store_short v[4:5], v1
-; GFX8-NEXT: flat_store_dword v[2:3], v0
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: freeze_v3bf16:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-SDAG-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX7-SDAG-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
+; GFX7-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: freeze_v3bf16:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX7-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-GISEL-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: buffer_store_short v4, v[2:3], s[4:7], 0 addr64 offset:2
+; GFX7-GISEL-NEXT: buffer_store_short v1, v[2:3], s[4:7], 0 addr64 offset:4
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GFX803-SDAG-LABEL: freeze_v3bf16:
+; GFX8-GFX803-SDAG: ; %bb.0:
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 4, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_store_short v[4:5], v1
+; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[2:3], v0
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: freeze_v3bf16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-GISEL-NEXT: v_add_u32_e32 v4, vcc, 2, v2
+; GFX8-GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GFX8-GISEL-NEXT: v_add_u32_e32 v6, vcc, 4, v2
+; GFX8-GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX8-GISEL-NEXT: flat_store_short v[2:3], v0
+; GFX8-GISEL-NEXT: flat_store_short v[4:5], v8
+; GFX8-GISEL-NEXT: flat_store_short v[6:7], v1
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GFX900-SDAG-LABEL: freeze_v3bf16:
; GFX8-GFX900-SDAG: ; %bb.0:
@@ -14139,48 +14277,120 @@ define void @freeze_v2i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
}
define void @freeze_v3i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
-; GFX6-LABEL: freeze_v3i8:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
-; GFX6-NEXT: buffer_store_byte v1, v[2:3], s[4:7], 0 addr64 offset:2
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX7-LABEL: freeze_v3i8:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
-; GFX7-NEXT: buffer_store_byte v1, v[2:3], s[4:7], 0 addr64 offset:2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: freeze_v3i8:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v0, v[0:1]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX8-NEXT: flat_store_short v[2:3], v0
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 2, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: flat_store_byte v[0:1], v4
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: freeze_v3i8:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX6-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-SDAG-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: buffer_store_byte v1, v[2:3], s[4:7], 0 addr64 offset:2
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: freeze_v3i8:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX6-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-GISEL-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: buffer_store_byte v1, v[2:3], s[4:7], 0 addr64 offset:2
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-SDAG-LABEL: freeze_v3i8:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX7-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: buffer_store_byte v1, v[2:3], s[4:7], 0 addr64 offset:2
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: freeze_v3i8:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX7-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: buffer_store_byte v1, v[2:3], s[4:7], 0 addr64 offset:2
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GFX803-SDAG-LABEL: freeze_v3i8:
+; GFX8-GFX803-SDAG: ; %bb.0:
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-GFX803-SDAG-NEXT: flat_store_short v[2:3], v0
+; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 2, v2
+; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_store_byte v[0:1], v4
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: freeze_v3i8:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v5, 0xff
+; GFX8-GISEL-NEXT: v_add_u32_e32 v0, vcc, 2, v2
+; GFX8-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v6, 8, v4
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v5, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX8-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX8-GISEL-NEXT: v_lshlrev_b16_e32 v6, 8, v6
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v4, v4, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX8-GISEL-NEXT: flat_store_short v[2:3], v4
+; GFX8-GISEL-NEXT: flat_store_byte v[0:1], v5
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GFX900-SDAG-LABEL: freeze_v3i8:
; GFX8-GFX900-SDAG: ; %bb.0:
@@ -14203,8 +14413,8 @@ define void @freeze_v3i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v4, 8, v4
; GFX9-GISEL-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
-; GFX9-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-GISEL-NEXT: global_store_short v[2:3], v0, off
; GFX9-GISEL-NEXT: global_store_byte_d16_hi v[2:3], v0, off offset:2
@@ -14230,8 +14440,8 @@ define void @freeze_v3i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX10-GISEL-NEXT: v_and_b32_sdwa v4, v4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-GISEL-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
-; GFX10-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX10-GISEL-NEXT: global_store_short v[2:3], v0, off
; GFX10-GISEL-NEXT: global_store_byte_d16_hi v[2:3], v0, off offset:2
@@ -14259,8 +14469,8 @@ define void @freeze_v3i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
; GFX11-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_bfe_u32 v1, v4, 0, 16
-; GFX11-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
@@ -14273,65 +14483,185 @@ define void @freeze_v3i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
}
define void @freeze_v4i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
-; GFX6-LABEL: freeze_v4i8:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX6-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: freeze_v4i8:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX6-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: freeze_v4i8:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX6-GISEL-LABEL: freeze_v4i8:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX6-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v5
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: freeze_v4i8:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v0, v[0:1]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: flat_store_dword v[2:3], v0
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: freeze_v4i8:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX7-SDAG-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: freeze_v4i8:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_store_dword v[2:3], v0, off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: freeze_v4i8:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX7-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v5
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: freeze_v4i8:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v[0:1], off
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dword v[2:3], v0, off
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GFX803-SDAG-LABEL: freeze_v4i8:
+; GFX8-GFX803-SDAG: ; %bb.0:
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[2:3], v0
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: freeze_v4i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: global_store_b32 v[2:3], v0, off
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: freeze_v4i8:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v6, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v0, v0, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-GISEL-NEXT: flat_store_dword v[2:3], v0
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GFX900-SDAG-LABEL: freeze_v4i8:
+; GFX8-GFX900-SDAG: ; %bb.0:
+; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX900-SDAG-NEXT: global_load_dword v0, v[0:1], off
+; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX900-SDAG-NEXT: global_store_dword v[2:3], v0, off
+; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: freeze_v4i8:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_dword v0, v[0:1], off
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v6, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v4, v0, v1, v4
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v0, v0, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v4, v6, v0
+; GFX9-GISEL-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: freeze_v4i8:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: global_load_dword v0, v[0:1], off
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: freeze_v4i8:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: global_load_dword v0, v[0:1], off
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 8
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 0xff
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v4, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, 0xff, v0, v1
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v0, v0, v5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v1, v4, v0
+; GFX10-GISEL-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: freeze_v4i8:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: freeze_v4i8:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX11-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v5
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v4, v1
+; GFX11-GISEL-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%a = load <4 x i8>, ptr addrspace(1) %ptra
%freeze = freeze <4 x i8> %a
store <4 x i8> %freeze, ptr addrspace(1) %ptrb
@@ -14339,65 +14669,241 @@ define void @freeze_v4i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
}
define void @freeze_v8i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
-; GFX6-LABEL: freeze_v8i8:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX6-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: freeze_v8i8:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX6-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: freeze_v8i8:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX6-GISEL-LABEL: freeze_v8i8:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v7, 8, v1
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v1
+; GFX6-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v6, 24, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v9, 24, v1
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX6-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX6-GISEL-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v7, 8, v7
+; GFX6-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX6-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v1, v7
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v9, 24, v9
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v1, v8
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: freeze_v8i8:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: freeze_v8i8:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX7-SDAG-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: freeze_v8i8:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: freeze_v8i8:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX7-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v7, 8, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v6, 24, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v9, 24, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v7, 8, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v7
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v9, 24, v9
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v8
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX7-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: freeze_v8i8:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GFX803-SDAG-LABEL: freeze_v8i8:
+; GFX8-GFX803-SDAG: ; %bb.0:
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: freeze_v8i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: freeze_v8i8:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v6, 8, v0
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v7, 8, v1
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v6, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v8, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v9, v0, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v10, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v4, v1, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v10
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v9
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX8-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GFX900-SDAG-LABEL: freeze_v8i8:
+; GFX8-GFX900-SDAG: ; %bb.0:
+; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: freeze_v8i8:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v6, 8, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v7, 8, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v6, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v8, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v9, v0, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v10, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v11, v1, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v4, v6
+; GFX9-GISEL-NEXT: v_and_or_b32 v1, v1, v4, v5
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v8, v9
+; GFX9-GISEL-NEXT: v_or3_b32 v1, v1, v10, v11
+; GFX9-GISEL-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: freeze_v8i8:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: freeze_v8i8:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 0xff
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v6, 8, v0
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v7, 8, v1
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v8, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v9, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v6, v4, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v4, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v7, v0, v5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v5, v1, v5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v6
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, 0xff, v1, v4
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v8, v7
+; GFX10-GISEL-NEXT: v_or3_b32 v1, v1, v9, v5
+; GFX10-GISEL-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: freeze_v8i8:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: freeze_v8i8:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v7, 8, v1
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v6, 24, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v1
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v9, 24, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX11-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX11-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX11-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX11-GISEL-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX11-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v7, 8, v7
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v9, 24, v9
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v4
+; GFX11-GISEL-NEXT: v_and_or_b32 v1, 0xff, v1, v7
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v5, v6
+; GFX11-GISEL-NEXT: v_or3_b32 v1, v1, v8, v9
+; GFX11-GISEL-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%a = load <8 x i8>, ptr addrspace(1) %ptra
%freeze = freeze <8 x i8> %a
store <8 x i8> %freeze, ptr addrspace(1) %ptrb
@@ -14405,65 +14911,353 @@ define void @freeze_v8i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
}
define void @freeze_v16i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
-; GFX6-LABEL: freeze_v16i8:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s4, s6
-; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX6-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: freeze_v16i8:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX6-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX6-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX7-LABEL: freeze_v16i8:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX6-GISEL-LABEL: freeze_v16i8:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX6-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-GISEL-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v0, 8, v4
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v9, 8, v5
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v12, 8, v6
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v15, 8, v7
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v13, 16, v6
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v16, 16, v7
+; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX6-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX6-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v12
+; GFX6-GISEL-NEXT: v_and_b32_e32 v15, 0xff, v15
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v8, 24, v4
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v11, 24, v5
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v14, 24, v6
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v17, 24, v7
+; GFX6-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX6-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX6-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX6-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX6-GISEL-NEXT: v_and_b32_e32 v10, 0xff, v10
+; GFX6-GISEL-NEXT: v_and_b32_e32 v13, 0xff, v13
+; GFX6-GISEL-NEXT: v_and_b32_e32 v16, 0xff, v16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v12, 8, v12
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v15, 8, v15
+; GFX6-GISEL-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX6-GISEL-NEXT: v_and_b32_e32 v11, 0xff, v11
+; GFX6-GISEL-NEXT: v_and_b32_e32 v14, 0xff, v14
+; GFX6-GISEL-NEXT: v_and_b32_e32 v17, 0xff, v17
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v4, v5, v9
+; GFX6-GISEL-NEXT: v_or_b32_e32 v5, v6, v12
+; GFX6-GISEL-NEXT: v_or_b32_e32 v6, v7, v15
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v8, 24, v8
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v11, 24, v11
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v14, 24, v14
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v17, 24, v17
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v4, v10
+; GFX6-GISEL-NEXT: v_or_b32_e32 v7, v5, v13
+; GFX6-GISEL-NEXT: v_or_b32_e32 v9, v6, v16
+; GFX6-GISEL-NEXT: v_or_b32_e32 v4, v0, v8
+; GFX6-GISEL-NEXT: v_or_b32_e32 v5, v1, v11
+; GFX6-GISEL-NEXT: v_or_b32_e32 v6, v7, v14
+; GFX6-GISEL-NEXT: v_or_b32_e32 v7, v9, v17
+; GFX6-GISEL-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: freeze_v16i8:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: freeze_v16i8:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
+; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
+; GFX7-SDAG-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: freeze_v16i8:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: freeze_v16i8:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: s_mov_b32 s6, 0
+; GFX7-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-GISEL-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v0, 8, v4
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v9, 8, v5
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v12, 8, v6
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v15, 8, v7
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v13, 16, v6
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v16, 16, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX7-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v15, 0xff, v15
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v8, 24, v4
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v11, 24, v5
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v14, 24, v6
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v17, 24, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v10, 0xff, v10
+; GFX7-GISEL-NEXT: v_and_b32_e32 v13, 0xff, v13
+; GFX7-GISEL-NEXT: v_and_b32_e32 v16, 0xff, v16
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v12, 8, v12
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v15, 8, v15
+; GFX7-GISEL-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX7-GISEL-NEXT: v_and_b32_e32 v11, 0xff, v11
+; GFX7-GISEL-NEXT: v_and_b32_e32 v14, 0xff, v14
+; GFX7-GISEL-NEXT: v_and_b32_e32 v17, 0xff, v17
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v4, v5, v9
+; GFX7-GISEL-NEXT: v_or_b32_e32 v5, v6, v12
+; GFX7-GISEL-NEXT: v_or_b32_e32 v6, v7, v15
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v8, 24, v8
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v11, 24, v11
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v14, 24, v14
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v17, 24, v17
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v4, v10
+; GFX7-GISEL-NEXT: v_or_b32_e32 v7, v5, v13
+; GFX7-GISEL-NEXT: v_or_b32_e32 v9, v6, v16
+; GFX7-GISEL-NEXT: v_or_b32_e32 v4, v0, v8
+; GFX7-GISEL-NEXT: v_or_b32_e32 v5, v1, v11
+; GFX7-GISEL-NEXT: v_or_b32_e32 v6, v7, v14
+; GFX7-GISEL-NEXT: v_or_b32_e32 v7, v9, v17
+; GFX7-GISEL-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: freeze_v16i8:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GFX803-SDAG-LABEL: freeze_v16i8:
+; GFX8-GFX803-SDAG: ; %bb.0:
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: freeze_v16i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b128 v[4:7], v[0:1], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: global_store_b128 v[2:3], v[4:7], off
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: freeze_v16i8:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 8
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, 0xff
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v9, 8, v5
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v10, 8, v6
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v11, 8, v7
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v8, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v9, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v10, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v12, v4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v13, v4, v0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v14, v5, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v15, v5, v0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v16, v6, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v17, v6, v0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v18, v7, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v4, v4, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v5, v5, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v6, v6, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v0, v7, v0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX8-GISEL-NEXT: v_or_b32_e32 v5, v5, v14
+; GFX8-GISEL-NEXT: v_or_b32_e32 v6, v6, v16
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v18
+; GFX8-GISEL-NEXT: v_or_b32_e32 v4, v4, v13
+; GFX8-GISEL-NEXT: v_or_b32_e32 v5, v5, v15
+; GFX8-GISEL-NEXT: v_or_b32_e32 v6, v6, v17
+; GFX8-GISEL-NEXT: v_or_b32_e32 v7, v1, v0
+; GFX8-GISEL-NEXT: flat_store_dwordx4 v[2:3], v[4:7]
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GFX900-SDAG-LABEL: freeze_v16i8:
+; GFX8-GFX900-SDAG: ; %bb.0:
+; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GFX900-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX900-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: freeze_v16i8:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 8
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xff
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v9, 8, v5
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v10, 8, v6
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v11, 8, v7
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v8, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v9, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v10, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v12, v4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v13, v4, v0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v14, v5, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v15, v5, v0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v16, v6, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v17, v6, v0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v18, v7, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v19, v7, v0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v4, v4, v0, v8
+; GFX9-GISEL-NEXT: v_and_or_b32 v5, v5, v0, v9
+; GFX9-GISEL-NEXT: v_and_or_b32 v6, v6, v0, v10
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v7, v0, v1
+; GFX9-GISEL-NEXT: v_or3_b32 v4, v4, v12, v13
+; GFX9-GISEL-NEXT: v_or3_b32 v5, v5, v14, v15
+; GFX9-GISEL-NEXT: v_or3_b32 v6, v6, v16, v17
+; GFX9-GISEL-NEXT: v_or3_b32 v7, v0, v18, v19
+; GFX9-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: freeze_v16i8:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: freeze_v16i8:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 8
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v9, 8, v5
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v10, 8, v6
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v11, 8, v7
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v12, v4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v8, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v9, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v10, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v0, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v13, v4, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v14, v5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v15, v5, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v16, v6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v17, v6, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v18, v7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v1, v7, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_or_b32 v4, 0xff, v4, v8
+; GFX10-GISEL-NEXT: v_and_or_b32 v5, 0xff, v5, v9
+; GFX10-GISEL-NEXT: v_and_or_b32 v6, 0xff, v6, v10
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v7, v0
+; GFX10-GISEL-NEXT: v_or3_b32 v4, v4, v12, v13
+; GFX10-GISEL-NEXT: v_or3_b32 v5, v5, v14, v15
+; GFX10-GISEL-NEXT: v_or3_b32 v6, v6, v16, v17
+; GFX10-GISEL-NEXT: v_or3_b32 v7, v0, v18, v1
+; GFX10-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: freeze_v16i8:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: global_store_b128 v[2:3], v[4:7], off
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: freeze_v16i8:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: global_load_b128 v[4:7], v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 8, v4
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v9, 8, v5
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v12, 8, v6
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v15, 8, v7
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v8, 24, v4
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v11, 24, v5
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v13, 16, v6
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v14, 24, v6
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v16, 16, v7
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v17, 24, v7
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX11-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v12
+; GFX11-GISEL-NEXT: v_and_b32_e32 v15, 0xff, v15
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX11-GISEL-NEXT: v_and_b32_e32 v10, 0xff, v10
+; GFX11-GISEL-NEXT: v_and_b32_e32 v11, 0xff, v11
+; GFX11-GISEL-NEXT: v_and_b32_e32 v13, 0xff, v13
+; GFX11-GISEL-NEXT: v_and_b32_e32 v14, 0xff, v14
+; GFX11-GISEL-NEXT: v_and_b32_e32 v16, 0xff, v16
+; GFX11-GISEL-NEXT: v_and_b32_e32 v17, 0xff, v17
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v12, 8, v12
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v15, 8, v15
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v8, 24, v8
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v11, 24, v11
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v14, 24, v14
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v17, 24, v17
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v4, v0
+; GFX11-GISEL-NEXT: v_and_or_b32 v5, 0xff, v5, v9
+; GFX11-GISEL-NEXT: v_and_or_b32 v6, 0xff, v6, v12
+; GFX11-GISEL-NEXT: v_and_or_b32 v7, 0xff, v7, v15
+; GFX11-GISEL-NEXT: v_or3_b32 v4, v0, v1, v8
+; GFX11-GISEL-NEXT: v_or3_b32 v5, v5, v10, v11
+; GFX11-GISEL-NEXT: v_or3_b32 v6, v6, v13, v14
+; GFX11-GISEL-NEXT: v_or3_b32 v7, v7, v16, v17
+; GFX11-GISEL-NEXT: global_store_b128 v[2:3], v[4:7], off
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%a = load <16 x i8>, ptr addrspace(1) %ptra
%freeze = freeze <16 x i8> %a
store <16 x i8> %freeze, ptr addrspace(1) %ptrb
diff --git a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
index b69fc139343c9..a754da7d00499 100644
--- a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
@@ -122,21 +122,28 @@ define i16 @test_v7i16_load_store(ptr addrspace(1) %ptr1, ptr addrspace(1) %ptr2
; GCN-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GCN-GISEL-FAKE16-NEXT: global_load_b128 v[4:7], v[0:1], off
; GCN-GISEL-FAKE16-NEXT: global_load_b128 v[8:11], v[2:3], off
-; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[12:13], 0
+; GCN-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], 0
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[12:13], 2
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[14:15], 4
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[16:17], 6
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[18:19], 8
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[20:21], 10
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[22:23], 12
; GCN-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GCN-GISEL-FAKE16-NEXT: s_wait_xcnt 0x1
; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v1, v6, v10
-; GCN-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
-; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v3, v5, v9
-; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v5, v7, v11
-; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[6:7], 12
-; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[10:11], 8
+; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v4, v4, v8
+; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v5, v5, v9
+; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v6, v7, v11
+; GCN-GISEL-FAKE16-NEXT: s_clause 0x6
+; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[2:3], v4, off
+; GCN-GISEL-FAKE16-NEXT: global_store_d16_hi_b16 v[12:13], v4, off
+; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[14:15], v5, off
+; GCN-GISEL-FAKE16-NEXT: global_store_d16_hi_b16 v[16:17], v5, off
+; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[18:19], v1, off
+; GCN-GISEL-FAKE16-NEXT: global_store_d16_hi_b16 v[20:21], v1, off
+; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[22:23], v6, off
; GCN-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v2, v4, v8
-; GCN-GISEL-FAKE16-NEXT: s_clause 0x2
-; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[6:7], v5, off
-; GCN-GISEL-FAKE16-NEXT: global_store_b32 v[10:11], v1, off
-; GCN-GISEL-FAKE16-NEXT: global_store_b64 v[12:13], v[2:3], off
; GCN-GISEL-FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
; GCN-SDAG-REAL16-LABEL: test_v7i16_load_store:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
index a2cbda17b41cd..83619ccbbedef 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
@@ -5713,19 +5713,34 @@ define <2 x half> @v_exp_fabs_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp_fabs_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v1, |v0|
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: v_exp_f32_e32 v0, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp_fabs_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e64 v1, |v0|
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp_fabs_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp_fabs_v2f16:
; SI: ; %bb.0:
@@ -5772,19 +5787,34 @@ define <2 x half> @v_exp_fneg_fabs_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp_fneg_fabs_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v1, -|v0|
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: v_exp_f32_e32 v0, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp_fneg_fabs_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e64 v1, -|v0|
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp_fneg_fabs_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp_fneg_fabs_v2f16:
; SI: ; %bb.0:
@@ -5833,19 +5863,34 @@ define <2 x half> @v_exp_fneg_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp_fneg_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v1, -v0
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: v_exp_f32_e32 v0, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp_fneg_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e64 v1, -v0
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp_fneg_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp_fneg_v2f16:
; SI: ; %bb.0:
@@ -5890,15 +5935,26 @@ define <2 x half> @v_exp_v2f16_fast(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp_v2f16_fast:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: s_movk_i32 s4, 0x3dc5
-; GFX900-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
-; GFX900-NEXT: v_exp_f16_e32 v1, v0
-; GFX900-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp_v2f16_fast:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: s_movk_i32 s4, 0x3dc5
+; GFX900-SDAG-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v1, v0
+; GFX900-SDAG-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp_v2f16_fast:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3dc5
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v2, 0x3dc5, v0
+; GFX900-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v2, v2
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp_v2f16_fast:
; SI: ; %bb.0:
@@ -5948,23 +6004,42 @@ define <3 x half> @v_exp_v3f16(<3 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp_v3f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v2, v0
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_mul_f32_e32 v2, 0x3fb8aa3b, v2
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; GFX900-NEXT: v_exp_f32_e32 v2, v2
-; GFX900-NEXT: v_exp_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v2, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp_v3f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v2, 0x3fb8aa3b, v2
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp_v3f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v2, 0x3fb8aa3b, v2
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp_v3f16:
; SI: ; %bb.0:
@@ -6013,18 +6088,32 @@ define <3 x half> @v_exp_v3f16_afn(<3 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp_v3f16_afn:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: s_movk_i32 s4, 0x3dc5
-; GFX900-NEXT: v_mul_f16_e32 v2, 0x3dc5, v0
-; GFX900-NEXT: v_mul_f16_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX900-NEXT: v_exp_f16_e32 v2, v2
-; GFX900-NEXT: v_exp_f16_e32 v0, v0
-; GFX900-NEXT: v_mul_f16_e32 v1, 0x3dc5, v1
-; GFX900-NEXT: v_exp_f16_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v2, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp_v3f16_afn:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: s_movk_i32 s4, 0x3dc5
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v2, 0x3dc5, v0
+; GFX900-SDAG-NEXT: v_mul_f16_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v2, v2
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v1, 0x3dc5, v1
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp_v3f16_afn:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x3dc5
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v3, 0x3dc5, v0
+; GFX900-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v1, 0x3dc5, v1
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v3, v3
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v3
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp_v3f16_afn:
; SI: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
index 260274c90a54a..a9ce3a7130396 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
@@ -5799,19 +5799,34 @@ define <2 x half> @v_exp10_fabs_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp10_fabs_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v1, |v0|
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: v_exp_f32_e32 v0, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp10_fabs_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e64 v1, |v0|
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp10_fabs_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp10_fabs_v2f16:
; SI: ; %bb.0:
@@ -5858,19 +5873,34 @@ define <2 x half> @v_exp10_fneg_fabs_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp10_fneg_fabs_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v1, -|v0|
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: v_exp_f32_e32 v0, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp10_fneg_fabs_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e64 v1, -|v0|
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp10_fneg_fabs_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp10_fneg_fabs_v2f16:
; SI: ; %bb.0:
@@ -5919,19 +5949,34 @@ define <2 x half> @v_exp10_fneg_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp10_fneg_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v1, -v0
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: v_exp_f32_e32 v0, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp10_fneg_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e64 v1, -v0
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp10_fneg_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp10_fneg_v2f16:
; SI: ; %bb.0:
@@ -5983,23 +6028,41 @@ define <2 x half> @v_exp10_v2f16_fast(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp10_v2f16_fast:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: s_movk_i32 s4, 0x113c
-; GFX900-NEXT: s_movk_i32 s5, 0x42a4
-; GFX900-NEXT: v_mul_f16_e32 v1, 0x113c, v0
-; GFX900-NEXT: v_mul_f16_e32 v2, 0x42a4, v0
-; GFX900-NEXT: v_mul_f16_sdwa v3, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX900-NEXT: v_mul_f16_sdwa v0, v0, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX900-NEXT: v_exp_f16_e32 v1, v1
-; GFX900-NEXT: v_exp_f16_e32 v2, v2
-; GFX900-NEXT: v_exp_f16_e32 v3, v3
-; GFX900-NEXT: v_exp_f16_e32 v0, v0
-; GFX900-NEXT: v_mul_f16_e32 v1, v2, v1
-; GFX900-NEXT: v_mul_f16_e32 v0, v0, v3
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp10_v2f16_fast:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: s_movk_i32 s4, 0x113c
+; GFX900-SDAG-NEXT: s_movk_i32 s5, 0x42a4
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v1, 0x113c, v0
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v2, 0x42a4, v0
+; GFX900-SDAG-NEXT: v_mul_f16_sdwa v3, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX900-SDAG-NEXT: v_mul_f16_sdwa v0, v0, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v2, v2
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v3, v3
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v1, v2, v1
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v0, v0, v3
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp10_v2f16_fast:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x113c
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x42a4
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v2, 0x113c, v0
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v4, 0x42a4, v0
+; GFX900-GISEL-NEXT: v_mul_f16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX900-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v2, v2
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v4, v4
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v2, v4, v2
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp10_v2f16_fast:
; SI: ; %bb.0:
@@ -6055,23 +6118,42 @@ define <3 x half> @v_exp10_v3f16(<3 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp10_v3f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v2, v0
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_mul_f32_e32 v2, 0x40549a78, v2
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; GFX900-NEXT: v_exp_f32_e32 v2, v2
-; GFX900-NEXT: v_exp_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; GFX900-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v2, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp10_v3f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v2, 0x40549a78, v2
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp10_v3f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v2, 0x40549a78, v2
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp10_v3f16:
; SI: ; %bb.0:
@@ -6130,28 +6212,52 @@ define <3 x half> @v_exp10_v3f16_afn(<3 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp10_v3f16_afn:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: s_movk_i32 s4, 0x113c
-; GFX900-NEXT: s_movk_i32 s5, 0x42a4
-; GFX900-NEXT: v_mul_f16_e32 v2, 0x113c, v1
-; GFX900-NEXT: v_mul_f16_e32 v1, 0x42a4, v1
-; GFX900-NEXT: v_mul_f16_e32 v3, 0x113c, v0
-; GFX900-NEXT: v_mul_f16_e32 v4, 0x42a4, v0
-; GFX900-NEXT: v_mul_f16_sdwa v5, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX900-NEXT: v_mul_f16_sdwa v0, v0, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX900-NEXT: v_exp_f16_e32 v2, v2
-; GFX900-NEXT: v_exp_f16_e32 v1, v1
-; GFX900-NEXT: v_exp_f16_e32 v3, v3
-; GFX900-NEXT: v_exp_f16_e32 v4, v4
-; GFX900-NEXT: v_exp_f16_e32 v5, v5
-; GFX900-NEXT: v_exp_f16_e32 v0, v0
-; GFX900-NEXT: v_mul_f16_e32 v1, v1, v2
-; GFX900-NEXT: v_mul_f16_e32 v2, v4, v3
-; GFX900-NEXT: v_mul_f16_e32 v0, v0, v5
-; GFX900-NEXT: v_pack_b32_f16 v0, v2, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp10_v3f16_afn:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: s_movk_i32 s4, 0x113c
+; GFX900-SDAG-NEXT: s_movk_i32 s5, 0x42a4
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v2, 0x113c, v1
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v1, 0x42a4, v1
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v3, 0x113c, v0
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v4, 0x42a4, v0
+; GFX900-SDAG-NEXT: v_mul_f16_sdwa v5, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX900-SDAG-NEXT: v_mul_f16_sdwa v0, v0, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v2, v2
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v3, v3
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v4, v4
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v5, v5
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v1, v1, v2
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v2, v4, v3
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v0, v0, v5
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp10_v3f16_afn:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x113c
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v4, 0x42a4
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v3, 0x113c, v0
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v5, 0x42a4, v0
+; GFX900-GISEL-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX900-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v4, 0x113c, v1
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v1, 0x42a4, v1
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v3, v3
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v5, v5
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v2, v2
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v4, v4
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v3, v5, v3
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v0, v0, v2
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v3, v0
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v1, v1, v4
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; SI-LABEL: v_exp10_v3f16_afn:
; SI: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
index 5148489812151..6268eec23a7c7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
@@ -3094,13 +3094,21 @@ define <2 x half> @v_exp2_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp2_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_exp_f16_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp2_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp2_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp2_v2f16:
; R600: ; %bb.0:
@@ -3138,13 +3146,22 @@ define <2 x half> @v_exp2_fabs_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp2_fabs_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_exp_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_exp_f16_e64 v0, |v0|
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp2_fabs_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_exp_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_exp_f16_e64 v0, |v0|
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp2_fabs_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp2_fabs_v2f16:
; R600: ; %bb.0:
@@ -3184,13 +3201,22 @@ define <2 x half> @v_exp2_fneg_fabs_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp2_fneg_fabs_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_exp_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_exp_f16_e64 v0, -|v0|
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp2_fneg_fabs_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_exp_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_exp_f16_e64 v0, -|v0|
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp2_fneg_fabs_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp2_fneg_fabs_v2f16:
; R600: ; %bb.0:
@@ -3231,13 +3257,22 @@ define <2 x half> @v_exp2_fneg_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp2_fneg_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_exp_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_exp_f16_e64 v0, -v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp2_fneg_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_exp_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_exp_f16_e64 v0, -v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp2_fneg_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp2_fneg_v2f16:
; R600: ; %bb.0:
@@ -3276,13 +3311,21 @@ define <2 x half> @v_exp2_v2f16_fast(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp2_v2f16_fast:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_exp_f16_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp2_v2f16_fast:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp2_v2f16_fast:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp2_v2f16_fast:
; R600: ; %bb.0:
@@ -3324,14 +3367,24 @@ define <3 x half> @v_exp_v3f16(<3 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v2
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp_v3f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_exp_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_exp_f16_e32 v0, v0
-; GFX900-NEXT: v_exp_f16_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp_v3f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_exp_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp_v3f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp_v3f16:
; R600: ; %bb.0:
@@ -3373,14 +3426,24 @@ define <3 x half> @v_exp2_v3f16_afn(<3 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v2
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_exp2_v3f16_afn:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_exp_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_exp_f16_e32 v0, v0
-; GFX900-NEXT: v_exp_f16_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_exp2_v3f16_afn:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_exp_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_exp_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_exp2_v3f16_afn:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_exp_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp2_v3f16_afn:
; R600: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
index 4e8f11cf1aa1b..a84942e06b5a0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
@@ -3,7 +3,7 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=CHECK,SDAG %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=CHECK,GFX11-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=CHECK,GFX11-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=CHECK,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=CHECK,GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
@@ -981,10 +981,11 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> in
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GISEL-NEXT: s_pack_ll_b32_b16 s1, s0, s1
+; GISEL-NEXT: s_and_b32 s0, s1, 0xffff
+; GISEL-NEXT: s_lshr_b32 s1, s1, 16
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward:
@@ -1048,10 +1049,11 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GISEL-NEXT: s_pack_ll_b32_b16 s1, s0, s1
+; GISEL-NEXT: s_and_b32 s0, s1, 0xffff
+; GISEL-NEXT: s_lshr_b32 s1, s1, 16
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_fptrunc_round_v2f32_to_v2f16_downward:
@@ -1107,10 +1109,11 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_towardzero(<2 x float
; GISEL: ; %bb.0:
; GISEL-NEXT: v_cvt_pkrtz_f16_f32_e32 v0, s0, v0
; GISEL-NEXT: v_cvt_pkrtz_f16_f32_e32 v1, s1, v0
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GISEL-NEXT: s_pack_ll_b32_b16 s1, s0, s1
+; GISEL-NEXT: s_and_b32 s0, s1, 0xffff
+; GISEL-NEXT: s_lshr_b32 s1, s1, 16
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_fptrunc_round_v2f32_to_v2f16_towardzero:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll b/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
index 0c4fadfbfeab7..ce5426b4efd8e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
@@ -416,18 +416,18 @@ define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_frexp_v2f16_v2i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_frexp_mant_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_frexp_mant_f16_e32 v2, v0
-; GFX9-NEXT: v_pack_b32_f16 v3, v2, v1
-; GFX9-NEXT: v_frexp_exp_i16_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
-; GFX9-NEXT: v_bfe_i32 v2, v1, 0, 16
-; GFX9-NEXT: v_bfe_i32 v1, v0, 0, 16
-; GFX9-NEXT: v_mov_b32_e32 v0, v3
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_frexp_v2f16_v2i32:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_frexp_mant_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-SDAG-NEXT: v_frexp_mant_f16_e32 v2, v0
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v3, v2, v1
+; GFX9-SDAG-NEXT: v_frexp_exp_i16_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-SDAG-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
+; GFX9-SDAG-NEXT: v_bfe_i32 v2, v1, 0, 16
+; GFX9-SDAG-NEXT: v_bfe_i32 v1, v0, 0, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_frexp_v2f16_v2i32:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -491,6 +491,18 @@ define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
; GFX12-SDAG-FAKE16-NEXT: v_bfe_i32 v2, v4, 0, 16
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-GISEL-LABEL: test_frexp_v2f16_v2i32:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_frexp_mant_f16_e32 v3, v0
+; GFX9-GISEL-NEXT: v_frexp_exp_i16_f16_e32 v1, v0
+; GFX9-GISEL-NEXT: v_frexp_mant_f16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-GISEL-NEXT: v_frexp_exp_i16_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX9-GISEL-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX9-GISEL-NEXT: v_pack_b32_f16 v0, v3, v4
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_frexp_v2f16_v2i32:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -584,13 +596,13 @@ define <2 x half> @test_frexp_v2f16_v2i32_only_use_fract(<2 x half> %a) {
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_frexp_mant_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_frexp_mant_f16_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_frexp_mant_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-SDAG-NEXT: v_frexp_mant_f16_e32 v0, v0
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -634,6 +646,14 @@ define <2 x half> @test_frexp_v2f16_v2i32_only_use_fract(<2 x half> %a) {
; GFX12-SDAG-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-GISEL-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_frexp_mant_f16_e32 v1, v0
+; GFX9-GISEL-NEXT: v_frexp_mant_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -681,38 +701,38 @@ define <2 x half> @test_frexp_v2f16_v2i32_only_use_fract(<2 x half> %a) {
}
define <2 x i32> @test_frexp_v2f16_v2i32_only_use_exp(<2 x half> %a) {
-; GFX6-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: s_mov_b32 s4, 0x7f800000
-; GFX6-NEXT: v_frexp_exp_i32_f32_e32 v2, v1
-; GFX6-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s4
-; GFX6-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
-; GFX6-NEXT: v_frexp_exp_i32_f32_e32 v2, v0
-; GFX6-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; GFX6-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
+; GFX6-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v2, v1
+; GFX6-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s4
+; GFX6-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
+; GFX6-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v2, v0
+; GFX6-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s4
+; GFX6-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_frexp_exp_i16_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
-; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_frexp_exp_i16_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-SDAG-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
+; GFX8-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX8-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_frexp_exp_i16_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
-; GFX9-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_frexp_exp_i16_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-SDAG-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
+; GFX9-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX9-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -766,6 +786,41 @@ define <2 x i32> @test_frexp_v2f16_v2i32_only_use_exp(<2 x half> %a) {
; GFX12-SDAG-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 16
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX6-GISEL-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; GFX6-GISEL-NEXT: v_frexp_exp_i32_f32_e32 v0, v1
+; GFX6-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, v2
+; GFX6-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX6-GISEL-NEXT: v_frexp_exp_i32_f32_e32 v1, v3
+; GFX6-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, v2
+; GFX6-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_frexp_exp_i16_f16_e32 v1, v0
+; GFX8-GISEL-NEXT: v_bfe_i32 v2, v1, 0, 16
+; GFX8-GISEL-NEXT: v_frexp_exp_i16_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-GISEL-NEXT: v_bfe_i32 v1, v0, 0, 16
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_frexp_exp_i16_f16_e32 v1, v0
+; GFX9-GISEL-NEXT: v_bfe_i32 v2, v1, 0, 16
+; GFX9-GISEL-NEXT: v_frexp_exp_i16_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-GISEL-NEXT: v_bfe_i32 v1, v0, 0, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
index 6f73787cdf03d..3982ae68e1afa 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
@@ -1,16 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu7.04 < %s | FileCheck --check-prefixes=GFX7CHECK,GFX7SELDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.04 < %s | FileCheck --check-prefixes=GFX7CHECK,GFX7GLISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.04 < %s | FileCheck --check-prefixes=GFX7CHECK,GFX7GLISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03 < %s | FileCheck --check-prefixes=GFX8CHECK,GFX8SELDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03 < %s | FileCheck --check-prefixes=GFX8CHECK,GFX8GLISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03 < %s | FileCheck --check-prefixes=GFX8CHECK,GFX8GLISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.08 < %s | FileCheck --check-prefixes=GFX9CHECK,GFX9SELDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.08 < %s | FileCheck --check-prefixes=GFX9CHECK,GFX9GLISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.08 < %s | FileCheck --check-prefixes=GFX9CHECK,GFX9GLISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.31 < %s | FileCheck --check-prefixes=GFX10CHECK,GFX10SELDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.31 < %s | FileCheck --check-prefixes=GFX10CHECK,GFX10GLISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.31 < %s | FileCheck --check-prefixes=GFX10CHECK,GFX10GLISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11CHECK,GFX11SELDAG,GFX11SELDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11CHECK,GFX11SELDAG,GFX11SELDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11CHECK,GFX11GLISEL,GFX11GLISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11CHECK,GFX11GLISEL,GFX11GLISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11CHECK,GFX11GLISEL,GFX11GLISEL-FAKE16 %s
; FIXME: There are code size regressions in GlobalISel due to use of SGPRs and
; moving those SGPRs into VGPRs.
@@ -1329,46 +1329,93 @@ define i1 @not_isnan_f16(half %x) {
}
define <2 x i1> @isnan_v2f16(<2 x half> %x) nounwind {
-; GFX7CHECK-LABEL: isnan_v2f16:
-; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7CHECK-NEXT: v_bfe_u32 v1, v0, 16, 15
-; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7c00
-; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v1
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8CHECK-LABEL: isnan_v2f16:
-; GFX8CHECK: ; %bb.0:
-; GFX8CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8CHECK-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX8CHECK-NEXT: v_cmp_u_f16_e32 vcc, v1, v1
-; GFX8CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX8CHECK-NEXT: v_cmp_u_f16_e32 vcc, v0, v0
-; GFX8CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX8CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7SELDAG-LABEL: isnan_v2f16:
+; GFX7SELDAG: ; %bb.0:
+; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7SELDAG-NEXT: v_bfe_u32 v1, v0, 16, 15
+; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v1
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX7SELDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9CHECK-LABEL: isnan_v2f16:
-; GFX9CHECK: ; %bb.0:
-; GFX9CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9CHECK-NEXT: v_cmp_u_f16_sdwa s[4:5], v0, v0 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9CHECK-NEXT: v_cmp_u_f16_e32 vcc, v0, v0
-; GFX9CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
-; GFX9CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX9CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX7GLISEL-LABEL: isnan_v2f16:
+; GFX7GLISEL: ; %bb.0:
+; GFX7GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7GLISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX7GLISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7GLISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; GFX7GLISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v0, v2
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX7GLISEL-NEXT: v_cmp_gt_u32_e32 vcc, v1, v2
+; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10CHECK-LABEL: isnan_v2f16:
-; GFX10CHECK: ; %bb.0:
-; GFX10CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10CHECK-NEXT: v_cmp_u_f16_e32 vcc_lo, v0, v0
-; GFX10CHECK-NEXT: v_cmp_u_f16_sdwa s4, v0, v0 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX10CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4
-; GFX10CHECK-NEXT: v_mov_b32_e32 v0, v2
-; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX8SELDAG-LABEL: isnan_v2f16:
+; GFX8SELDAG: ; %bb.0:
+; GFX8SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8SELDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX8SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v1, v1
+; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v0, v0
+; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8GLISEL-LABEL: isnan_v2f16:
+; GFX8GLISEL: ; %bb.0:
+; GFX8GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8GLISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX8GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v0, 3
+; GFX8GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX8GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v1, 3
+; GFX8GLISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GFX8GLISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9SELDAG-LABEL: isnan_v2f16:
+; GFX9SELDAG: ; %bb.0:
+; GFX9SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9SELDAG-NEXT: v_cmp_u_f16_sdwa s[4:5], v0, v0 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v0, v0
+; GFX9SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GFX9SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9GLISEL-LABEL: isnan_v2f16:
+; GFX9GLISEL: ; %bb.0:
+; GFX9GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9GLISEL-NEXT: v_mov_b32_e32 v1, 3
+; GFX9GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v0, 3
+; GFX9GLISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX9GLISEL-NEXT: v_cmp_class_f16_sdwa s[4:5], v0, v1 src0_sel:WORD_1 src1_sel:DWORD
+; GFX9GLISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GFX9GLISEL-NEXT: v_mov_b32_e32 v0, v2
+; GFX9GLISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10SELDAG-LABEL: isnan_v2f16:
+; GFX10SELDAG: ; %bb.0:
+; GFX10SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10SELDAG-NEXT: v_cmp_u_f16_e32 vcc_lo, v0, v0
+; GFX10SELDAG-NEXT: v_cmp_u_f16_sdwa s4, v0, v0 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX10SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4
+; GFX10SELDAG-NEXT: v_mov_b32_e32 v0, v2
+; GFX10SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10GLISEL-LABEL: isnan_v2f16:
+; GFX10GLISEL: ; %bb.0:
+; GFX10GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10GLISEL-NEXT: v_mov_b32_e32 v1, 3
+; GFX10GLISEL-NEXT: v_cmp_class_f16_e64 s4, v0, 3
+; GFX10GLISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, s4
+; GFX10GLISEL-NEXT: v_cmp_class_f16_sdwa s4, v0, v1 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10GLISEL-NEXT: v_mov_b32_e32 v0, v2
+; GFX10GLISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4
+; GFX10GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11SELDAG-TRUE16-LABEL: isnan_v2f16:
; GFX11SELDAG-TRUE16: ; %bb.0:
@@ -1406,10 +1453,10 @@ define <2 x i1> @isnan_v2f16(<2 x half> %x) nounwind {
; GFX11GLISEL-FAKE16: ; %bb.0:
; GFX11GLISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11GLISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11GLISEL-FAKE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0, v0
-; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11GLISEL-FAKE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1, v1
-; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11GLISEL-FAKE16-NEXT: v_cmp_class_f16_e64 s0, v0, 3
+; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11GLISEL-FAKE16-NEXT: v_cmp_class_f16_e64 s0, v1, 3
+; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
; GFX11GLISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%1 = call <2 x i1> @llvm.is.fpclass.v2f16(<2 x half> %x, i32 3) ; nan
ret <2 x i1> %1
@@ -1450,42 +1497,83 @@ define <3 x i1> @isnan_v3f16(<3 x half> %x) nounwind {
; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8CHECK-LABEL: isnan_v3f16:
-; GFX8CHECK: ; %bb.0:
-; GFX8CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8CHECK-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX8CHECK-NEXT: v_cmp_u_f16_e32 vcc, v2, v2
-; GFX8CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX8CHECK-NEXT: v_cmp_u_f16_e32 vcc, v0, v0
-; GFX8CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX8CHECK-NEXT: v_cmp_u_f16_e32 vcc, v1, v1
-; GFX8CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX8CHECK-NEXT: v_mov_b32_e32 v1, v3
-; GFX8CHECK-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9CHECK-LABEL: isnan_v3f16:
-; GFX9CHECK: ; %bb.0:
-; GFX9CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9CHECK-NEXT: v_cmp_u_f16_sdwa s[4:5], v0, v0 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9CHECK-NEXT: v_cmp_u_f16_e32 vcc, v0, v0
-; GFX9CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5]
-; GFX9CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX9CHECK-NEXT: v_cmp_u_f16_e32 vcc, v1, v1
-; GFX9CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX9CHECK-NEXT: v_mov_b32_e32 v1, v3
-; GFX9CHECK-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10CHECK-LABEL: isnan_v3f16:
-; GFX10CHECK: ; %bb.0:
-; GFX10CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10CHECK-NEXT: v_cmp_u_f16_sdwa s4, v0, v0 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10CHECK-NEXT: v_cmp_u_f16_e32 vcc_lo, v0, v0
-; GFX10CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s4
-; GFX10CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX10CHECK-NEXT: v_cmp_u_f16_e32 vcc_lo, v1, v1
-; GFX10CHECK-NEXT: v_mov_b32_e32 v1, v3
-; GFX10CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX8SELDAG-LABEL: isnan_v3f16:
+; GFX8SELDAG: ; %bb.0:
+; GFX8SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8SELDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v2, v2
+; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX8SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v0, v0
+; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v1, v1
+; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8SELDAG-NEXT: v_mov_b32_e32 v1, v3
+; GFX8SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8GLISEL-LABEL: isnan_v3f16:
+; GFX8GLISEL: ; %bb.0:
+; GFX8GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8GLISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v0, 3
+; GFX8GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX8GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v2, 3
+; GFX8GLISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5]
+; GFX8GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v1, 3
+; GFX8GLISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX8GLISEL-NEXT: v_mov_b32_e32 v1, v3
+; GFX8GLISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9SELDAG-LABEL: isnan_v3f16:
+; GFX9SELDAG: ; %bb.0:
+; GFX9SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9SELDAG-NEXT: v_cmp_u_f16_sdwa s[4:5], v0, v0 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v0, v0
+; GFX9SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5]
+; GFX9SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v1, v1
+; GFX9SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9SELDAG-NEXT: v_mov_b32_e32 v1, v3
+; GFX9SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9GLISEL-LABEL: isnan_v3f16:
+; GFX9GLISEL: ; %bb.0:
+; GFX9GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9GLISEL-NEXT: v_mov_b32_e32 v2, 3
+; GFX9GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v0, 3
+; GFX9GLISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5]
+; GFX9GLISEL-NEXT: v_cmp_class_f16_sdwa s[4:5], v0, v2 src0_sel:WORD_1 src1_sel:DWORD
+; GFX9GLISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5]
+; GFX9GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v1, 3
+; GFX9GLISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX9GLISEL-NEXT: v_mov_b32_e32 v0, v4
+; GFX9GLISEL-NEXT: v_mov_b32_e32 v1, v3
+; GFX9GLISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10SELDAG-LABEL: isnan_v3f16:
+; GFX10SELDAG: ; %bb.0:
+; GFX10SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10SELDAG-NEXT: v_cmp_u_f16_sdwa s4, v0, v0 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10SELDAG-NEXT: v_cmp_u_f16_e32 vcc_lo, v0, v0
+; GFX10SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s4
+; GFX10SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10SELDAG-NEXT: v_cmp_u_f16_e32 vcc_lo, v1, v1
+; GFX10SELDAG-NEXT: v_mov_b32_e32 v1, v3
+; GFX10SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX10SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10GLISEL-LABEL: isnan_v3f16:
+; GFX10GLISEL: ; %bb.0:
+; GFX10GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10GLISEL-NEXT: v_mov_b32_e32 v2, 3
+; GFX10GLISEL-NEXT: v_cmp_class_f16_e64 s4, v0, 3
+; GFX10GLISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, s4
+; GFX10GLISEL-NEXT: v_cmp_class_f16_sdwa s4, v0, v2 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10GLISEL-NEXT: v_mov_b32_e32 v0, v4
+; GFX10GLISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, s4
+; GFX10GLISEL-NEXT: v_cmp_class_f16_e64 s4, v1, 3
+; GFX10GLISEL-NEXT: v_mov_b32_e32 v1, v3
+; GFX10GLISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, s4
+; GFX10GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11SELDAG-TRUE16-LABEL: isnan_v3f16:
; GFX11SELDAG-TRUE16: ; %bb.0:
@@ -1532,13 +1620,13 @@ define <3 x i1> @isnan_v3f16(<3 x half> %x) nounwind {
; GFX11GLISEL-FAKE16: ; %bb.0:
; GFX11GLISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11GLISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11GLISEL-FAKE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0, v0
-; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11GLISEL-FAKE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v2, v2
-; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; GFX11GLISEL-FAKE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1, v1
+; GFX11GLISEL-FAKE16-NEXT: v_cmp_class_f16_e64 s0, v0, 3
+; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11GLISEL-FAKE16-NEXT: v_cmp_class_f16_e64 s0, v2, 3
+; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11GLISEL-FAKE16-NEXT: v_cmp_class_f16_e64 s0, v1, 3
; GFX11GLISEL-FAKE16-NEXT: v_mov_b32_e32 v1, v3
-; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
; GFX11GLISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%1 = call <3 x i1> @llvm.is.fpclass.v3f16(<3 x half> %x, i32 3) ; nan
ret <3 x i1> %1
@@ -1587,49 +1675,97 @@ define <4 x i1> @isnan_v4f16(<4 x half> %x) nounwind {
; GFX7GLISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
; GFX7GLISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8CHECK-LABEL: isnan_v4f16:
-; GFX8CHECK: ; %bb.0:
-; GFX8CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8CHECK-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX8CHECK-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX8CHECK-NEXT: v_cmp_u_f16_e32 vcc, v3, v3
-; GFX8CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GFX8CHECK-NEXT: v_cmp_u_f16_e32 vcc, v2, v2
-; GFX8CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX8CHECK-NEXT: v_cmp_u_f16_e32 vcc, v0, v0
-; GFX8CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX8CHECK-NEXT: v_cmp_u_f16_e32 vcc, v1, v1
-; GFX8CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX8CHECK-NEXT: v_mov_b32_e32 v1, v4
-; GFX8CHECK-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9CHECK-LABEL: isnan_v4f16:
-; GFX9CHECK: ; %bb.0:
-; GFX9CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9CHECK-NEXT: v_cmp_u_f16_sdwa s[4:5], v0, v0 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9CHECK-NEXT: v_cmp_u_f16_e32 vcc, v0, v0
-; GFX9CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5]
-; GFX9CHECK-NEXT: v_cmp_u_f16_sdwa s[4:5], v1, v1 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX9CHECK-NEXT: v_cmp_u_f16_e32 vcc, v1, v1
-; GFX9CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5]
-; GFX9CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX9CHECK-NEXT: v_mov_b32_e32 v1, v4
-; GFX9CHECK-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10CHECK-LABEL: isnan_v4f16:
-; GFX10CHECK: ; %bb.0:
-; GFX10CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10CHECK-NEXT: v_cmp_u_f16_sdwa s4, v0, v0 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10CHECK-NEXT: v_cmp_u_f16_e32 vcc_lo, v0, v0
-; GFX10CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s4
-; GFX10CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX10CHECK-NEXT: v_cmp_u_f16_e32 vcc_lo, v1, v1
-; GFX10CHECK-NEXT: v_cmp_u_f16_sdwa s4, v1, v1 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10CHECK-NEXT: v_mov_b32_e32 v1, v4
-; GFX10CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX10CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s4
-; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX8SELDAG-LABEL: isnan_v4f16:
+; GFX8SELDAG: ; %bb.0:
+; GFX8SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8SELDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX8SELDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX8SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v3, v3
+; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX8SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v2, v2
+; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX8SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v0, v0
+; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v1, v1
+; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8SELDAG-NEXT: v_mov_b32_e32 v1, v4
+; GFX8SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8GLISEL-LABEL: isnan_v4f16:
+; GFX8GLISEL: ; %bb.0:
+; GFX8GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8GLISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v0, 3
+; GFX8GLISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX8GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v2, 3
+; GFX8GLISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX8GLISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5]
+; GFX8GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v1, 3
+; GFX8GLISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX8GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v3, 3
+; GFX8GLISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5]
+; GFX8GLISEL-NEXT: v_mov_b32_e32 v1, v4
+; GFX8GLISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9SELDAG-LABEL: isnan_v4f16:
+; GFX9SELDAG: ; %bb.0:
+; GFX9SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9SELDAG-NEXT: v_cmp_u_f16_sdwa s[4:5], v0, v0 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v0, v0
+; GFX9SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5]
+; GFX9SELDAG-NEXT: v_cmp_u_f16_sdwa s[4:5], v1, v1 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9SELDAG-NEXT: v_cmp_u_f16_e32 vcc, v1, v1
+; GFX9SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5]
+; GFX9SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9SELDAG-NEXT: v_mov_b32_e32 v1, v4
+; GFX9SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9GLISEL-LABEL: isnan_v4f16:
+; GFX9GLISEL: ; %bb.0:
+; GFX9GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9GLISEL-NEXT: v_mov_b32_e32 v3, 3
+; GFX9GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v0, 3
+; GFX9GLISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5]
+; GFX9GLISEL-NEXT: v_cmp_class_f16_sdwa s[4:5], v0, v3 src0_sel:WORD_1 src1_sel:DWORD
+; GFX9GLISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[4:5]
+; GFX9GLISEL-NEXT: v_cmp_class_f16_e64 s[4:5], v1, 3
+; GFX9GLISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX9GLISEL-NEXT: v_cmp_class_f16_sdwa s[4:5], v1, v3 src0_sel:WORD_1 src1_sel:DWORD
+; GFX9GLISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5]
+; GFX9GLISEL-NEXT: v_mov_b32_e32 v0, v4
+; GFX9GLISEL-NEXT: v_mov_b32_e32 v1, v5
+; GFX9GLISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10SELDAG-LABEL: isnan_v4f16:
+; GFX10SELDAG: ; %bb.0:
+; GFX10SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10SELDAG-NEXT: v_cmp_u_f16_sdwa s4, v0, v0 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10SELDAG-NEXT: v_cmp_u_f16_e32 vcc_lo, v0, v0
+; GFX10SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s4
+; GFX10SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10SELDAG-NEXT: v_cmp_u_f16_e32 vcc_lo, v1, v1
+; GFX10SELDAG-NEXT: v_cmp_u_f16_sdwa s4, v1, v1 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10SELDAG-NEXT: v_mov_b32_e32 v1, v4
+; GFX10SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX10SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s4
+; GFX10SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10GLISEL-LABEL: isnan_v4f16:
+; GFX10GLISEL: ; %bb.0:
+; GFX10GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10GLISEL-NEXT: v_mov_b32_e32 v3, 3
+; GFX10GLISEL-NEXT: v_cmp_class_f16_e64 s4, v0, 3
+; GFX10GLISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, s4
+; GFX10GLISEL-NEXT: v_cmp_class_f16_sdwa s4, v0, v3 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10GLISEL-NEXT: v_mov_b32_e32 v0, v4
+; GFX10GLISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, s4
+; GFX10GLISEL-NEXT: v_cmp_class_f16_e64 s4, v1, 3
+; GFX10GLISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, s4
+; GFX10GLISEL-NEXT: v_cmp_class_f16_sdwa s4, v1, v3 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10GLISEL-NEXT: v_mov_b32_e32 v1, v5
+; GFX10GLISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, s4
+; GFX10GLISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11SELDAG-TRUE16-LABEL: isnan_v4f16:
; GFX11SELDAG-TRUE16: ; %bb.0:
@@ -1682,16 +1818,16 @@ define <4 x i1> @isnan_v4f16(<4 x half> %x) nounwind {
; GFX11GLISEL-FAKE16-LABEL: isnan_v4f16:
; GFX11GLISEL-FAKE16: ; %bb.0:
; GFX11GLISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11GLISEL-FAKE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0, v0
-; GFX11GLISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX11GLISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11GLISEL-FAKE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1, v1
-; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX11GLISEL-FAKE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v4, v4
-; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX11GLISEL-FAKE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v3, v3
-; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11GLISEL-FAKE16-NEXT: v_cmp_class_f16_e64 s0, v0, 3
+; GFX11GLISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11GLISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11GLISEL-FAKE16-NEXT: v_cmp_class_f16_e64 s0, v1, 3
+; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11GLISEL-FAKE16-NEXT: v_cmp_class_f16_e64 s0, v3, 3
+; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11GLISEL-FAKE16-NEXT: v_cmp_class_f16_e64 s0, v4, 3
+; GFX11GLISEL-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
; GFX11GLISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%1 = call <4 x i1> @llvm.is.fpclass.v4f16(<4 x half> %x, i32 3) ; nan
ret <4 x i1> %1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll b/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll
index ab6050b4b0b67..2502760abb912 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll
@@ -432,17 +432,17 @@ define <2 x half> @test_ldexp_v2f16_v2i32(<2 x half> %a, <2 x i32> %b) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_ldexp_v2f16_v2i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x8000
-; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fff
-; GFX9-NEXT: v_med3_i32 v2, v2, s4, v3
-; GFX9-NEXT: v_med3_i32 v1, v1, s4, v3
-; GFX9-NEXT: v_ldexp_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_ldexp_f16_e32 v0, v0, v1
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_ldexp_v2f16_v2i32:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-SDAG-NEXT: v_med3_i32 v2, v2, s4, v3
+; GFX9-SDAG-NEXT: v_med3_i32 v1, v1, s4, v3
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_ldexp_v2f16_v2i32:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -470,6 +470,18 @@ define <2 x half> @test_ldexp_v2f16_v2i32(<2 x half> %a, <2 x i32> %b) {
; GFX11-SDAG-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-GISEL-LABEL: test_ldexp_v2f16_v2i32:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xffff8000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-GISEL-NEXT: v_med3_i32 v1, v1, v3, v4
+; GFX9-GISEL-NEXT: v_med3_i32 v2, v2, v3, v4
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v1, v0, v1
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_ldexp_v2f16_v2i32:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -485,15 +497,17 @@ define <2 x half> @test_ldexp_v2f16_v2i32(<2 x half> %a, <2 x i32> %b) {
; GFX11-GISEL-FAKE16-LABEL: test_ldexp_v2f16_v2i32:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v2, v2, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, v1, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v2, v3, v2
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, 0xffff8000, v1, v3
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v2, 0xffff8000, v2, v3
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v4, v2
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> %a, <2 x i32> %b)
ret <2 x half> %result
@@ -524,13 +538,13 @@ define <2 x half> @test_ldexp_v2f16_v2i16(<2 x half> %a, <2 x i16> %b) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_ldexp_v2f16_v2i16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_ldexp_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_ldexp_f16_e32 v0, v0, v1
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_ldexp_v2f16_v2i16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_ldexp_v2f16_v2i16:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -550,6 +564,14 @@ define <2 x half> @test_ldexp_v2f16_v2i16(<2 x half> %a, <2 x i16> %b) {
; GFX11-SDAG-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-GISEL-LABEL: test_ldexp_v2f16_v2i16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v2, v0, v1
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_ldexp_v2f16_v2i16:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -560,12 +582,14 @@ define <2 x half> @test_ldexp_v2f16_v2i16(<2 x half> %a, <2 x i16> %b) {
; GFX11-GISEL-FAKE16-LABEL: test_ldexp_v2f16_v2i16:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v2, v3, v2
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v2, v3
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> %a, <2 x i16> %b)
ret <2 x half> %result
@@ -603,19 +627,19 @@ define <3 x half> @test_ldexp_v3f16_v3i32(<3 x half> %a, <3 x i32> %b) {
; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_ldexp_v3f16_v3i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x8000
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fff
-; GFX9-NEXT: v_med3_i32 v3, v3, s4, v5
-; GFX9-NEXT: v_med3_i32 v2, v2, s4, v5
-; GFX9-NEXT: v_med3_i32 v4, v4, s4, v5
-; GFX9-NEXT: v_ldexp_f16_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_ldexp_f16_e32 v0, v0, v2
-; GFX9-NEXT: v_ldexp_f16_e32 v1, v1, v4
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v3
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_ldexp_v3f16_v3i32:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX9-SDAG-NEXT: v_med3_i32 v3, v3, s4, v5
+; GFX9-SDAG-NEXT: v_med3_i32 v2, v2, s4, v5
+; GFX9-SDAG-NEXT: v_med3_i32 v4, v4, s4, v5
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v1, v1, v4
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_ldexp_v3f16_v3i32:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -647,6 +671,21 @@ define <3 x half> @test_ldexp_v3f16_v3i32(<3 x half> %a, <3 x i32> %b) {
; GFX11-SDAG-FAKE16-NEXT: v_ldexp_f16_e32 v1, v1, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-GISEL-LABEL: test_ldexp_v3f16_v3i32:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 0xffff8000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0x7fff
+; GFX9-GISEL-NEXT: v_med3_i32 v2, v2, v5, v6
+; GFX9-GISEL-NEXT: v_med3_i32 v3, v3, v5, v6
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v2, v0, v2
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_med3_i32 v3, v4, v5, v6
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_ldexp_v3f16_v3i32:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -664,17 +703,22 @@ define <3 x half> @test_ldexp_v3f16_v3i32(<3 x half> %a, <3 x i32> %b) {
; GFX11-GISEL-FAKE16-LABEL: test_ldexp_v3f16_v3i32:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v3, v3, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v2, v2, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v3, v5, v3
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v2, 0xffff8000, v2, v5
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v4, 0xffff8000, v4, v5
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v3, 0xffff8000, v3, v5
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v2
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v2, v4, s0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v1, v4
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v2, v6, v3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v3
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> %a, <3 x i32> %b)
ret <3 x half> %result
@@ -710,14 +754,14 @@ define <3 x half> @test_ldexp_v3f16_v3i16(<3 x half> %a, <3 x i16> %b) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_ldexp_v3f16_v3i16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_ldexp_f16_sdwa v4, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_ldexp_f16_e32 v0, v0, v2
-; GFX9-NEXT: v_ldexp_f16_e32 v1, v1, v3
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_ldexp_v3f16_v3i16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v4, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_ldexp_v3f16_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -739,6 +783,16 @@ define <3 x half> @test_ldexp_v3f16_v3i16(<3 x half> %a, <3 x i16> %b) {
; GFX11-SDAG-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-GISEL-LABEL: test_ldexp_v3f16_v3i16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v4, v0, v2
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v4
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_ldexp_v3f16_v3i16:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -750,13 +804,18 @@ define <3 x half> @test_ldexp_v3f16_v3i16(<3 x half> %a, <3 x i16> %b) {
; GFX11-GISEL-FAKE16-LABEL: test_ldexp_v3f16_v3i16:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v2
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v1, v3
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v2, v5, v4
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v2, v4, v5
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> %a, <3 x i16> %b)
ret <3 x half> %result
@@ -803,22 +862,22 @@ define <4 x half> @test_ldexp_v4f16_v4i32(<4 x half> %a, <4 x i32> %b) {
; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_ldexp_v4f16_v4i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x8000
-; GFX9-NEXT: v_mov_b32_e32 v6, 0x7fff
-; GFX9-NEXT: v_med3_i32 v5, v5, s4, v6
-; GFX9-NEXT: v_med3_i32 v4, v4, s4, v6
-; GFX9-NEXT: v_med3_i32 v3, v3, s4, v6
-; GFX9-NEXT: v_med3_i32 v2, v2, s4, v6
-; GFX9-NEXT: v_ldexp_f16_sdwa v5, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_ldexp_f16_e32 v1, v1, v4
-; GFX9-NEXT: v_ldexp_f16_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_ldexp_f16_e32 v0, v0, v2
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v3
-; GFX9-NEXT: v_pack_b32_f16 v1, v1, v5
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_ldexp_v4f16_v4i32:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v6, 0x7fff
+; GFX9-SDAG-NEXT: v_med3_i32 v5, v5, s4, v6
+; GFX9-SDAG-NEXT: v_med3_i32 v4, v4, s4, v6
+; GFX9-SDAG-NEXT: v_med3_i32 v3, v3, s4, v6
+; GFX9-SDAG-NEXT: v_med3_i32 v2, v2, s4, v6
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v5, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v1, v1, v4
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v3
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v1, v1, v5
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_ldexp_v4f16_v4i32:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -858,6 +917,23 @@ define <4 x half> @test_ldexp_v4f16_v4i32(<4 x half> %a, <4 x i32> %b) {
; GFX11-SDAG-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v5
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-GISEL-LABEL: test_ldexp_v4f16_v4i32:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0xffff8000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX9-GISEL-NEXT: v_med3_i32 v2, v2, v6, v7
+; GFX9-GISEL-NEXT: v_med3_i32 v3, v3, v6, v7
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v2, v0, v2
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_med3_i32 v3, v4, v6, v7
+; GFX9-GISEL-NEXT: v_med3_i32 v4, v5, v6, v7
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v3, v1, v3
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v1, 16, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_ldexp_v4f16_v4i32:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -878,22 +954,26 @@ define <4 x half> @test_ldexp_v4f16_v4i32(<4 x half> %a, <4 x i32> %b) {
; GFX11-GISEL-FAKE16-LABEL: test_ldexp_v4f16_v4i32:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v5, v5, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v3, v3, s0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v6, 0x7fff
; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v2, v2, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v4, v4, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v5, v6, v5
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v3, v7, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v2, 0xffff8000, v2, v6
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v4, 0xffff8000, v4, v6
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v3, 0xffff8000, v3, v6
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v5, 0xffff8000, v5, v6
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v1, v4
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v3
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v2, v7, v3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v3, v8, v5
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v2, 16, v0
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v5
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> %a, <4 x i32> %b)
ret <4 x half> %result
@@ -938,16 +1018,16 @@ define <4 x half> @test_ldexp_v4f16_v4i16(<4 x half> %a, <4 x i16> %b) {
; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_ldexp_v4f16_v4i16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_ldexp_f16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_ldexp_f16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_ldexp_f16_e32 v1, v1, v3
-; GFX9-NEXT: v_ldexp_f16_e32 v0, v0, v2
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v5
-; GFX9-NEXT: v_pack_b32_f16 v1, v1, v4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_ldexp_v4f16_v4i16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v5
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v1, v1, v4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_ldexp_v4f16_v4i16:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -975,6 +1055,17 @@ define <4 x half> @test_ldexp_v4f16_v4i16(<4 x half> %a, <4 x i16> %b) {
; GFX11-SDAG-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v3
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-GISEL-LABEL: test_ldexp_v4f16_v4i16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v4, v0, v2
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v2, v1, v3
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v4
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_ldexp_v4f16_v4i16:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -987,18 +1078,21 @@ define <4 x half> @test_ldexp_v4f16_v4i16(<4 x half> %a, <4 x i16> %b) {
; GFX11-GISEL-FAKE16-LABEL: test_ldexp_v4f16_v4i16:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v1, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v1, v3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v2, v4, v6
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v3, v5, v7
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v2, v6, v5
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v3, v7, v4
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> %a, <4 x i16> %b)
ret <4 x half> %result
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log.ll b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
index d131bc2955457..b0623c84528a3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
@@ -4676,17 +4676,18 @@ define <2 x half> @v_log_v2f16(<2 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log_v2f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3f317218
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v1, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v0, v2, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_v2f16:
@@ -4733,19 +4734,34 @@ define <2 x half> @v_log_fabs_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log_fabs_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v1, |v0|
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f32_e32 v1, v1
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log_fabs_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e64 v1, |v0|
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log_fabs_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log_fabs_v2f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -4796,17 +4812,20 @@ define <2 x half> @v_log_fabs_v2f16(<2 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log_fabs_v2f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v1, |v0|
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v1, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v0, v2, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_fabs_v2f16:
@@ -4855,19 +4874,34 @@ define <2 x half> @v_log_fneg_fabs_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log_fneg_fabs_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v1, -|v0|
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f32_e32 v1, v1
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log_fneg_fabs_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e64 v1, -|v0|
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log_fneg_fabs_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log_fneg_fabs_v2f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -4918,17 +4952,20 @@ define <2 x half> @v_log_fneg_fabs_v2f16(<2 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log_fneg_fabs_v2f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v1, -|v0|
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v1, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v0, v2, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_fneg_fabs_v2f16:
@@ -4978,19 +5015,34 @@ define <2 x half> @v_log_fneg_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log_fneg_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v1, -v0
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f32_e32 v1, v1
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log_fneg_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e64 v1, -v0
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log_fneg_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log_fneg_v2f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -5041,17 +5093,20 @@ define <2 x half> @v_log_fneg_v2f16(<2 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log_fneg_v2f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v1, -v0
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v1, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v0, v2, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_fneg_v2f16:
@@ -5204,23 +5259,42 @@ define <3 x half> @v_log_v3f16(<3 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log_v3f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v2, v0
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_log_f32_e32 v2, v2
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_log_f32_e32 v1, v1
-; GFX900-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v2, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log_v3f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log_v3f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log_v3f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -5281,21 +5355,25 @@ define <3 x half> @v_log_v3f16(<3 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log_v3f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3f317218
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v2
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v3, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_3)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v2, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v2
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, v3, 0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v0, v3, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v3, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v2, v2, v3, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_v3f16:
@@ -5345,17 +5423,30 @@ define <3 x half> @v_log_v3f16_fast(<3 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log_v3f16_fast:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_e32 v2, v0
-; GFX900-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v1, v1
-; GFX900-NEXT: v_mul_f16_e32 v2, 0x398c, v2
-; GFX900-NEXT: v_mul_f16_e32 v0, 0x398c, v0
-; GFX900-NEXT: v_mul_f16_e32 v1, 0x398c, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v2, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log_v3f16_fast:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_e32 v2, v0
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v2, 0x398c, v2
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log_v3f16_fast:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v2, 0x398c, v2
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log_v3f16_fast:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -5403,16 +5494,18 @@ define <3 x half> @v_log_v3f16_fast(<3 x half> %in) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_3)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v2, v2
-; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, 0x398c, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, 0x398c, v0
; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v2, 0x398c, v2
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_v3f16_fast:
@@ -5479,28 +5572,51 @@ define <4 x half> @v_log_v4f16(<4 x half> %in) {
; VI-NEXT: v_or_b32_e32 v1, v2, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log_v4f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v2, v1
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v3, v0
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f32_e32 v2, v2
-; GFX900-NEXT: v_log_f32_e32 v1, v1
-; GFX900-NEXT: v_log_f32_e32 v3, v3
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; GFX900-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX900-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v3, v0
-; GFX900-NEXT: v_pack_b32_f16 v1, v2, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log_v4f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v3, v3
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v3, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v1, v2, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log_v4f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_log_f32_e32 v3, v3
+; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v1, v3, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log_v4f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -5568,24 +5684,27 @@ define <4 x half> @v_log_v4f16(<4 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log_v4f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3f317218
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v2
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v3, v3
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, 0x3f317218
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v4, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v5, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v2, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_3)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v3, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v3, v3
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v4, 0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, v4, 0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v0, v4, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v1, v5, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v2, v2, v4, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v3, v3, v4, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v3
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_v4f16:
@@ -5645,20 +5764,35 @@ define <4 x half> @v_log_v4f16_fast(<4 x half> %in) {
; VI-NEXT: v_or_b32_e32 v1, v2, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log_v4f16_fast:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_e32 v2, v1
-; GFX900-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v3, v0
-; GFX900-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_mul_f16_e32 v2, 0x398c, v2
-; GFX900-NEXT: v_mul_f16_e32 v1, 0x398c, v1
-; GFX900-NEXT: v_mul_f16_e32 v3, 0x398c, v3
-; GFX900-NEXT: v_mul_f16_e32 v0, 0x398c, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v3, v0
-; GFX900-NEXT: v_pack_b32_f16 v1, v2, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log_v4f16_fast:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_e32 v2, v1
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v3, v0
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v2, 0x398c, v2
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v3, 0x398c, v3
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v3, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v1, v2, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log_v4f16_fast:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f16_e32 v3, v1
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v2, 0x398c, v2
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v3, 0x398c, v3
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v1, v3, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log_v4f16_fast:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -5716,14 +5850,14 @@ define <4 x half> @v_log_v4f16_fast(<4 x half> %in) {
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v2, v2
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v3, v3
-; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v1, 0x398c, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v1, 0x398c, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v2, 0x398c, v2
; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v3, 0x398c, v3
@@ -5842,15 +5976,25 @@ define <2 x half> @v_log_v2f16_fpmath(<2 x half> %x) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log_v2f16_fpmath:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v0, v0
-; GFX900-NEXT: s_movk_i32 s4, 0x398c
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX900-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log_v2f16_fpmath:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: s_movk_i32 s4, 0x398c
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX900-SDAG-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log_v2f16_fpmath:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x398c398c
+; GFX900-GISEL-NEXT: v_pk_mul_f16 v0, v0, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log_v2f16_fpmath:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -5891,7 +6035,7 @@ define <2 x half> @v_log_v2f16_fpmath(<2 x half> %x) {
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, 0x398c, v0 op_sel_hi:[0,1]
+; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, 0x398c398c, v0
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_v2f16_fpmath:
@@ -5941,19 +6085,33 @@ define <3 x half> @v_log_v3f16_fpmath(<3 x half> %x) {
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log_v3f16_fpmath:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v0, v0
-; GFX900-NEXT: v_log_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v1, v1
-; GFX900-NEXT: s_movk_i32 s4, 0x398c
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX900-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
-; GFX900-NEXT: v_pack_b32_f16 v1, v1, v3
-; GFX900-NEXT: v_pk_mul_f16 v1, v1, s4
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log_v3f16_fpmath:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: s_movk_i32 s4, 0x398c
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX900-SDAG-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v1, v1, v3
+; GFX900-SDAG-NEXT: v_pk_mul_f16 v1, v1, s4
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log_v3f16_fpmath:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x398c398c
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: v_pk_mul_f16 v0, v0, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x398c
+; GFX900-GISEL-NEXT: v_pk_mul_f16 v1, v1, v2
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log_v3f16_fpmath:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -6000,17 +6158,16 @@ define <3 x half> @v_log_v3f16_fpmath(<3 x half> %x) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_3)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v2, v2
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v3, v3
+; GFX1100-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v3
+; GFX1100-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, 0x398c, v0 op_sel_hi:[0,1]
+; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, 0x398c398c, v0
; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v1, 0x398c, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6071,19 +6228,33 @@ define <4 x half> @v_log_v4f16_fpmath(<4 x half> %x) {
; VI-NEXT: v_or_b32_e32 v1, v2, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log_v4f16_fpmath:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v0, v0
-; GFX900-NEXT: v_log_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v1, v1
-; GFX900-NEXT: s_movk_i32 s4, 0x398c
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX900-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
-; GFX900-NEXT: v_pack_b32_f16 v1, v1, v3
-; GFX900-NEXT: v_pk_mul_f16 v1, v1, s4 op_sel_hi:[1,0]
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log_v4f16_fpmath:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: s_movk_i32 s4, 0x398c
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX900-SDAG-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v1, v1, v3
+; GFX900-SDAG-NEXT: v_pk_mul_f16 v1, v1, s4 op_sel_hi:[1,0]
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log_v4f16_fpmath:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f16_e32 v3, v1
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x398c398c
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v1, v3, v1
+; GFX900-GISEL-NEXT: v_pk_mul_f16 v0, v0, v2
+; GFX900-GISEL-NEXT: v_pk_mul_f16 v1, v1, v2
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log_v4f16_fpmath:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -6133,8 +6304,8 @@ define <4 x half> @v_log_v4f16_fpmath(<4 x half> %x) {
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v2, v2
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v3, v3
@@ -6142,8 +6313,8 @@ define <4 x half> @v_log_v4f16_fpmath(<4 x half> %x) {
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v3
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, 0x398c, v0 op_sel_hi:[0,1]
-; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v1, 0x398c, v1 op_sel_hi:[0,1]
+; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, 0x398c398c, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v1, 0x398c398c, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_v4f16_fpmath:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
index a2e60c075b352..2f5beb4864b47 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
@@ -4676,17 +4676,18 @@ define <2 x half> @v_log10_v2f16(<2 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log10_v2f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3e9a209b
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v1, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v0, v2, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_v2f16:
@@ -4733,19 +4734,34 @@ define <2 x half> @v_log10_fabs_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log10_fabs_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v1, |v0|
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f32_e32 v1, v1
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log10_fabs_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e64 v1, |v0|
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log10_fabs_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log10_fabs_v2f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -4796,17 +4812,20 @@ define <2 x half> @v_log10_fabs_v2f16(<2 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log10_fabs_v2f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v1, |v0|
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v1, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v0, v2, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_fabs_v2f16:
@@ -4855,19 +4874,34 @@ define <2 x half> @v_log10_fneg_fabs_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log10_fneg_fabs_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v1, -|v0|
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f32_e32 v1, v1
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log10_fneg_fabs_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e64 v1, -|v0|
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log10_fneg_fabs_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log10_fneg_fabs_v2f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -4918,17 +4952,20 @@ define <2 x half> @v_log10_fneg_fabs_v2f16(<2 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log10_fneg_fabs_v2f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v1, -|v0|
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v1, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v0, v2, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_fneg_fabs_v2f16:
@@ -4978,19 +5015,34 @@ define <2 x half> @v_log10_fneg_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log10_fneg_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e64 v1, -v0
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f32_e32 v1, v1
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log10_fneg_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e64 v1, -v0
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log10_fneg_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log10_fneg_v2f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -5041,17 +5093,20 @@ define <2 x half> @v_log10_fneg_v2f16(<2 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log10_fneg_v2f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v1, -v0
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v1, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v0, v2, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, v2, 0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_fneg_v2f16:
@@ -5204,23 +5259,42 @@ define <3 x half> @v_log10_v3f16(<3 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log10_v3f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v2, v0
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_log_f32_e32 v2, v2
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_log_f32_e32 v1, v1
-; GFX900-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v2, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log10_v3f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log10_v3f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log10_v3f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -5281,21 +5355,25 @@ define <3 x half> @v_log10_v3f16(<3 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log10_v3f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3e9a209b
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v2
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v3, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_3)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v2, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v2
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, v3, 0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v0, v3, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v3, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v2, v2, v3, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_v3f16:
@@ -5345,17 +5423,30 @@ define <3 x half> @v_log10_v3f16_fast(<3 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log10_v3f16_fast:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_e32 v2, v0
-; GFX900-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v1, v1
-; GFX900-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
-; GFX900-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
-; GFX900-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v2, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log10_v3f16_fast:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_e32 v2, v0
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log10_v3f16_fast:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log10_v3f16_fast:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -5403,16 +5494,18 @@ define <3 x half> @v_log10_v3f16_fast(<3 x half> %in) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_3)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v2, v2
-; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_v3f16_fast:
@@ -5479,28 +5572,51 @@ define <4 x half> @v_log10_v4f16(<4 x half> %in) {
; VI-NEXT: v_or_b32_e32 v1, v2, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log10_v4f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_e32 v2, v1
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v3, v0
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f32_e32 v2, v2
-; GFX900-NEXT: v_log_f32_e32 v1, v1
-; GFX900-NEXT: v_log_f32_e32 v3, v3
-; GFX900-NEXT: v_log_f32_e32 v0, v0
-; GFX900-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
-; GFX900-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; GFX900-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
-; GFX900-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX900-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX900-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v3, v0
-; GFX900-NEXT: v_pack_b32_f16 v1, v2, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log10_v4f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX900-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_log_f32_e32 v3, v3
+; GFX900-SDAG-NEXT: v_log_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v3, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v1, v2, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log10_v4f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX900-GISEL-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_log_f32_e32 v3, v3
+; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX900-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v1, v3, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log10_v4f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -5568,24 +5684,27 @@ define <4 x half> @v_log10_v4f16(<4 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log10_v4f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_mov_b32 s0, 0x3e9a209b
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v2
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v3, v3
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX1100-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, 0x3e9a209b
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v4, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v5, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v2, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_3)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v3, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v2, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v3, v3
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v4, 0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, v4, 0
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v0, v4, s0, 0
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixhi_f16 v1, v5, s0, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v2, v2, v4, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_fma_mixlo_f16 v3, v3, v4, 0
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v3
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_v4f16:
@@ -5645,20 +5764,35 @@ define <4 x half> @v_log10_v4f16_fast(<4 x half> %in) {
; VI-NEXT: v_or_b32_e32 v1, v2, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log10_v4f16_fast:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_e32 v2, v1
-; GFX900-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v3, v0
-; GFX900-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
-; GFX900-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
-; GFX900-NEXT: v_mul_f16_e32 v3, 0x34d1, v3
-; GFX900-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v3, v0
-; GFX900-NEXT: v_pack_b32_f16 v1, v2, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log10_v4f16_fast:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_e32 v2, v1
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v3, v0
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v3, 0x34d1, v3
+; GFX900-SDAG-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v3, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v1, v2, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log10_v4f16_fast:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f16_e32 v3, v1
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v3, 0x34d1, v3
+; GFX900-GISEL-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v1, v3, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log10_v4f16_fast:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -5716,14 +5850,14 @@ define <4 x half> @v_log10_v4f16_fast(<4 x half> %in) {
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v2, v2
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v3, v3
-; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
; GFX1100-GISEL-FAKE16-NEXT: v_mul_f16_e32 v3, 0x34d1, v3
@@ -5842,15 +5976,25 @@ define <2 x half> @v_log10_v2f16_fpmath(<2 x half> %x) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log10_v2f16_fpmath:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v0, v0
-; GFX900-NEXT: s_movk_i32 s4, 0x34d1
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX900-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log10_v2f16_fpmath:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: s_movk_i32 s4, 0x34d1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX900-SDAG-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log10_v2f16_fpmath:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x34d134d1
+; GFX900-GISEL-NEXT: v_pk_mul_f16 v0, v0, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log10_v2f16_fpmath:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -5891,7 +6035,7 @@ define <2 x half> @v_log10_v2f16_fpmath(<2 x half> %x) {
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, 0x34d1, v0 op_sel_hi:[0,1]
+; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, 0x34d134d1, v0
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_v2f16_fpmath:
@@ -5941,19 +6085,33 @@ define <3 x half> @v_log10_v3f16_fpmath(<3 x half> %x) {
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log10_v3f16_fpmath:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v0, v0
-; GFX900-NEXT: v_log_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v1, v1
-; GFX900-NEXT: s_movk_i32 s4, 0x34d1
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX900-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
-; GFX900-NEXT: v_pack_b32_f16 v1, v1, v3
-; GFX900-NEXT: v_pk_mul_f16 v1, v1, s4
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log10_v3f16_fpmath:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: s_movk_i32 s4, 0x34d1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX900-SDAG-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v1, v1, v3
+; GFX900-SDAG-NEXT: v_pk_mul_f16 v1, v1, s4
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log10_v3f16_fpmath:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x34d134d1
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: v_pk_mul_f16 v0, v0, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x34d1
+; GFX900-GISEL-NEXT: v_pk_mul_f16 v1, v1, v2
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log10_v3f16_fpmath:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -6000,17 +6158,16 @@ define <3 x half> @v_log10_v3f16_fpmath(<3 x half> %x) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_3)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v2, v2
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v3, v3
+; GFX1100-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v3
+; GFX1100-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, 0x34d1, v0 op_sel_hi:[0,1]
+; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, 0x34d134d1, v0
; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v1, 0x34d1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6071,19 +6228,33 @@ define <4 x half> @v_log10_v4f16_fpmath(<4 x half> %x) {
; VI-NEXT: v_or_b32_e32 v1, v2, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log10_v4f16_fpmath:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v0, v0
-; GFX900-NEXT: v_log_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v1, v1
-; GFX900-NEXT: s_movk_i32 s4, 0x34d1
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX900-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
-; GFX900-NEXT: v_pack_b32_f16 v1, v1, v3
-; GFX900-NEXT: v_pk_mul_f16 v1, v1, s4 op_sel_hi:[1,0]
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log10_v4f16_fpmath:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: s_movk_i32 s4, 0x34d1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX900-SDAG-NEXT: v_pk_mul_f16 v0, v0, s4 op_sel_hi:[1,0]
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v1, v1, v3
+; GFX900-SDAG-NEXT: v_pk_mul_f16 v1, v1, s4 op_sel_hi:[1,0]
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log10_v4f16_fpmath:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f16_e32 v3, v1
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x34d134d1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v1, v3, v1
+; GFX900-GISEL-NEXT: v_pk_mul_f16 v0, v0, v2
+; GFX900-GISEL-NEXT: v_pk_mul_f16 v1, v1, v2
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log10_v4f16_fpmath:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -6133,8 +6304,8 @@ define <4 x half> @v_log10_v4f16_fpmath(<4 x half> %x) {
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v2, v2
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v3, v3
@@ -6142,8 +6313,8 @@ define <4 x half> @v_log10_v4f16_fpmath(<4 x half> %x) {
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v3
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, 0x34d1, v0 op_sel_hi:[0,1]
-; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v1, 0x34d1, v1 op_sel_hi:[0,1]
+; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v0, 0x34d134d1, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_pk_mul_f16 v1, 0x34d134d1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_v4f16_fpmath:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
index c059eae6cc153..5efcbfcc443b3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -2611,13 +2611,21 @@ define <2 x half> @v_log2_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log2_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log2_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log2_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log2_v2f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -2691,13 +2699,22 @@ define <2 x half> @v_log2_fabs_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log2_fabs_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e64 v0, |v0|
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log2_fabs_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e64 v0, |v0|
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log2_fabs_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log2_fabs_v2f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -2729,10 +2746,11 @@ define <2 x half> @v_log2_fabs_v2f16(<2 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log2_fabs_v2f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e64 v0, |v0|
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e64 v1, |v1|
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -2775,13 +2793,22 @@ define <2 x half> @v_log2_fneg_fabs_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log2_fneg_fabs_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e64 v0, -|v0|
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log2_fneg_fabs_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e64 v0, -|v0|
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log2_fneg_fabs_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log2_fneg_fabs_v2f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -2813,10 +2840,11 @@ define <2 x half> @v_log2_fneg_fabs_v2f16(<2 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log2_fneg_fabs_v2f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-GISEL-FAKE16-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e64 v0, -|v0|
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e64 v1, -|v1|
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -2860,13 +2888,22 @@ define <2 x half> @v_log2_fneg_v2f16(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log2_fneg_v2f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e64 v0, -v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log2_fneg_v2f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e64 v0, -v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log2_fneg_v2f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log2_fneg_v2f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -2898,10 +2935,11 @@ define <2 x half> @v_log2_fneg_v2f16(<2 x half> %in) {
; GFX1100-GISEL-FAKE16-LABEL: v_log2_fneg_v2f16:
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-GISEL-FAKE16-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e64 v0, -v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e64 v1, -v1
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -2943,13 +2981,21 @@ define <2 x half> @v_log2_v2f16_fast(<2 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log2_v2f16_fast:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v0, v0
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log2_v2f16_fast:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log2_v2f16_fast:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log2_v2f16_fast:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -3027,14 +3073,24 @@ define <3 x half> @v_log2_v3f16(<3 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v2
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log2_v3f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v0, v0
-; GFX900-NEXT: v_log_f16_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log2_v3f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_log_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log2_v3f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log2_v3f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -3068,12 +3124,14 @@ define <3 x half> @v_log2_v3f16(<3 x half> %in) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_3)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v2, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log2_v3f16:
@@ -3116,14 +3174,24 @@ define <3 x half> @v_log2_v3f16_fast(<3 x half> %in) {
; VI-NEXT: v_or_b32_e32 v0, v0, v2
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log2_v3f16_fast:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v0, v0
-; GFX900-NEXT: v_log_f16_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v2
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log2_v3f16_fast:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_log_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log2_v3f16_fast:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f16_e32 v1, v1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log2_v3f16_fast:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -3157,12 +3225,14 @@ define <3 x half> @v_log2_v3f16_fast(<3 x half> %in) {
; GFX1100-GISEL-FAKE16: ; %bb.0:
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_3)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v2, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-GISEL-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
+; GFX1100-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log2_v3f16_fast:
@@ -3213,16 +3283,27 @@ define <4 x half> @v_log2_v4f16(<4 x half> %in) {
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log2_v4f16:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v0, v0
-; GFX900-NEXT: v_log_f16_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v3
-; GFX900-NEXT: v_pack_b32_f16 v1, v1, v2
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log2_v4f16:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_log_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v3
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v1, v1, v2
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log2_v4f16:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f16_e32 v3, v1
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v1, v3, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log2_v4f16:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -3262,8 +3343,8 @@ define <4 x half> @v_log2_v4f16(<4 x half> %in) {
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v2, v2
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v3, v3
@@ -3320,16 +3401,27 @@ define <4 x half> @v_log2_v4f16_fast(<4 x half> %in) {
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_log2_v4f16_fast:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_log_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_log_f16_e32 v0, v0
-; GFX900-NEXT: v_log_f16_e32 v1, v1
-; GFX900-NEXT: v_pack_b32_f16 v0, v0, v3
-; GFX900-NEXT: v_pack_b32_f16 v1, v1, v2
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; GFX900-SDAG-LABEL: v_log2_v4f16_fast:
+; GFX900-SDAG: ; %bb.0:
+; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-SDAG-NEXT: v_log_f16_e32 v0, v0
+; GFX900-SDAG-NEXT: v_log_f16_e32 v1, v1
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v0, v0, v3
+; GFX900-SDAG-NEXT: v_pack_b32_f16 v1, v1, v2
+; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-GISEL-LABEL: v_log2_v4f16_fast:
+; GFX900-GISEL: ; %bb.0:
+; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-GISEL-NEXT: v_log_f16_e32 v2, v0
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_log_f16_e32 v3, v1
+; GFX900-GISEL-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX900-GISEL-NEXT: v_pack_b32_f16 v1, v3, v1
+; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-SDAG-TRUE16-LABEL: v_log2_v4f16_fast:
; GFX1100-SDAG-TRUE16: ; %bb.0:
@@ -3369,8 +3461,8 @@ define <4 x half> @v_log2_v4f16_fast(<4 x half> %in) {
; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX1100-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v0, v0
+; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v1, v1
; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v2, v2
; GFX1100-GISEL-FAKE16-NEXT: v_log_f16_e32 v3, v3
@@ -3412,7 +3504,5 @@ attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memo
; GFX1100-SDAG: {{.*}}
; GFX689-GISEL: {{.*}}
; GFX689-SDAG: {{.*}}
-; GFX900-GISEL: {{.*}}
-; GFX900-SDAG: {{.*}}
; VI-GISEL: {{.*}}
; VI-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.modf.ll b/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
index bc980135e79fd..64154d25aa552 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
declare { half, half } @llvm.modf.f16(half)
declare { <2 x half>, <2 x half> } @llvm.modf.v2f16(<2 x half>)
@@ -82,61 +82,116 @@ define half @test_modf_f16_only_use_integer(half %x) {
}
define { <2 x half>, <2 x half> } @test_modf_v2f16(<2 x half> %x) {
-; GFX9-LABEL: test_modf_v2f16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x7c00
-; GFX9-NEXT: v_trunc_f16_e32 v1, v0
-; GFX9-NEXT: v_sub_f16_e32 v2, v0, v1
-; GFX9-NEXT: v_cmp_neq_f16_e64 vcc, |v0|, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; GFX9-NEXT: s_movk_i32 s5, 0x7fff
-; GFX9-NEXT: v_bfi_b32 v2, s5, v2, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_trunc_f16_e32 v3, v0
-; GFX9-NEXT: v_sub_f16_e32 v4, v0, v3
-; GFX9-NEXT: v_cmp_neq_f16_e64 vcc, |v0|, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; GFX9-NEXT: v_bfi_b32 v0, s5, v4, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v2, v0
-; GFX9-NEXT: v_pack_b32_f16 v1, v1, v3
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_modf_v2f16:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX9-SDAG-NEXT: v_trunc_f16_e32 v1, v0
+; GFX9-SDAG-NEXT: v_sub_f16_e32 v2, v0, v1
+; GFX9-SDAG-NEXT: v_cmp_neq_f16_e64 vcc, |v0|, s4
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX9-SDAG-NEXT: s_movk_i32 s5, 0x7fff
+; GFX9-SDAG-NEXT: v_bfi_b32 v2, s5, v2, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_trunc_f16_e32 v3, v0
+; GFX9-SDAG-NEXT: v_sub_f16_e32 v4, v0, v3
+; GFX9-SDAG-NEXT: v_cmp_neq_f16_e64 vcc, |v0|, s4
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX9-SDAG-NEXT: v_bfi_b32 v0, s5, v4, v0
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v1, v1, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_modf_v2f16:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_trunc_f16_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7c00
+; GFX9-GISEL-NEXT: v_sub_f16_e32 v2, v0, v1
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_e64 s[4:5], |v0|, v3
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v5, 0xffff8000, v0
+; GFX9-GISEL-NEXT: v_or_b32_e32 v2, v2, v5
+; GFX9-GISEL-NEXT: v_trunc_f16_sdwa v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-GISEL-NEXT: v_sub_f16_sdwa v6, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_sdwa s[4:5], |v0|, v3 src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xffff8000
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v3, v6, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0x7fff, v3
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-GISEL-NEXT: v_pack_b32_f16 v1, v1, v5
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { <2 x half>, <2 x half> } @llvm.modf.v2f16(<2 x half> %x)
ret { <2 x half>, <2 x half> } %result
}
define <2 x half> @test_modf_v2f16_only_use_fract(<2 x half> %x) {
-; GFX9-LABEL: test_modf_v2f16_only_use_fract:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x7c00
-; GFX9-NEXT: v_trunc_f16_e32 v1, v0
-; GFX9-NEXT: v_sub_f16_e32 v1, v0, v1
-; GFX9-NEXT: v_cmp_neq_f16_e64 vcc, |v0|, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX9-NEXT: s_movk_i32 s5, 0x7fff
-; GFX9-NEXT: v_bfi_b32 v1, s5, v1, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_trunc_f16_e32 v2, v0
-; GFX9-NEXT: v_sub_f16_e32 v2, v0, v2
-; GFX9-NEXT: v_cmp_neq_f16_e64 vcc, |v0|, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; GFX9-NEXT: v_bfi_b32 v0, s5, v2, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v1, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_modf_v2f16_only_use_fract:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7c00
+; GFX9-SDAG-NEXT: v_trunc_f16_e32 v1, v0
+; GFX9-SDAG-NEXT: v_sub_f16_e32 v1, v0, v1
+; GFX9-SDAG-NEXT: v_cmp_neq_f16_e64 vcc, |v0|, s4
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-SDAG-NEXT: s_movk_i32 s5, 0x7fff
+; GFX9-SDAG-NEXT: v_bfi_b32 v1, s5, v1, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_trunc_f16_e32 v2, v0
+; GFX9-SDAG-NEXT: v_sub_f16_e32 v2, v0, v2
+; GFX9-SDAG-NEXT: v_cmp_neq_f16_e64 vcc, |v0|, s4
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX9-SDAG-NEXT: v_bfi_b32 v0, s5, v2, v0
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_modf_v2f16_only_use_fract:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_trunc_f16_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x7c00
+; GFX9-GISEL-NEXT: v_sub_f16_e32 v1, v0, v1
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_e64 s[4:5], |v0|, v2
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v4, 0xffff8000, v0
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX9-GISEL-NEXT: v_trunc_f16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-GISEL-NEXT: v_sub_f16_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_sdwa s[4:5], |v0|, v2 src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xffff8000
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5]
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0x7fff, v2
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { <2 x half>, <2 x half> } @llvm.modf.v2f16(<2 x half> %x)
%result.0 = extractvalue { <2 x half>, <2 x half> } %result, 0
ret <2 x half> %result.0
}
define <2 x half> @test_modf_v2f16_only_use_integer(<2 x half> %x) {
-; GFX9-LABEL: test_modf_v2f16_only_use_integer:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_trunc_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_trunc_f16_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_modf_v2f16_only_use_integer:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_trunc_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-SDAG-NEXT: v_trunc_f16_e32 v0, v0
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_modf_v2f16_only_use_integer:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_trunc_f16_e32 v1, v0
+; GFX9-GISEL-NEXT: v_trunc_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { <2 x half>, <2 x half> } @llvm.modf.v2f16(<2 x half> %x)
%result.1 = extractvalue { <2 x half>, <2 x half> } %result, 1
ret <2 x half> %result.1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
index 4e9e00c96aa64..bcaba060b812f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
@@ -4,11 +4,11 @@
; RUN: llc -global-isel=0 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=+real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX11-TRUE16,GFX11-TRUE16-SDAG %s
; RUN: llc -global-isel=1 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=+real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX11-TRUE16,GFX11-TRUE16-GISEL %s
; RUN: llc -global-isel=0 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX11-FAKE16,GFX11-FAKE16-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX11-FAKE16,GFX11-FAKE16-GISEL %s
+; RUN: llc -global-isel=1 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX11-FAKE16,GFX11-FAKE16-GISEL %s
; RUN: llc -global-isel=0 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=+real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12-TRUE16,GFX12-TRUE16-SDAG %s
; RUN: llc -global-isel=1 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=+real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12-TRUE16,GFX12-TRUE16-GISEL %s
; RUN: llc -global-isel=0 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=-real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12-FAKE16,GFX12-FAKE16-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=-real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12-FAKE16,GFX12-FAKE16-GISEL %s
+; RUN: llc -global-isel=1 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu12.00 -mattr=-real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12-FAKE16,GFX12-FAKE16-GISEL %s
declare half @llvm.sqrt.f16(half %a)
declare <2 x half> @llvm.sqrt.v2f16(<2 x half> %a)
@@ -286,28 +286,47 @@ define amdgpu_kernel void @sqrt_v2f16(
; GFX11-TRUE16-GISEL-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-TRUE16-GISEL-NEXT: s_endpgm
;
-; GFX11-FAKE16-LABEL: sqrt_v2f16:
-; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-FAKE16-NEXT: s_mov_b32 s11, s7
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: v_sqrt_f16_e32 v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_sqrt_f16_e32 v1, v1
-; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX11-FAKE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-FAKE16-SDAG-LABEL: sqrt_v2f16:
+; GFX11-FAKE16-SDAG: ; %bb.0: ; %entry
+; GFX11-FAKE16-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s6, -1
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s10, s6
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s11, s7
+; GFX11-FAKE16-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s8, s2
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s9, s3
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX11-FAKE16-SDAG-NEXT: buffer_load_b32 v0, off, s[8:11], 0
+; GFX11-FAKE16-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-FAKE16-SDAG-NEXT: v_sqrt_f16_e32 v0, v0
+; GFX11-FAKE16-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-SDAG-NEXT: v_sqrt_f16_e32 v1, v1
+; GFX11-FAKE16-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-FAKE16-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX11-FAKE16-SDAG-NEXT: buffer_store_b32 v0, off, s[4:7], 0
+; GFX11-FAKE16-SDAG-NEXT: s_endpgm
+;
+; GFX11-FAKE16-GISEL-LABEL: sqrt_v2f16:
+; GFX11-FAKE16-GISEL: ; %bb.0: ; %entry
+; GFX11-FAKE16-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-GISEL-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-FAKE16-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-FAKE16-GISEL-NEXT: v_sqrt_f16_e32 v0, s2
+; GFX11-FAKE16-GISEL-NEXT: v_sqrt_f16_e32 v1, s3
+; GFX11-FAKE16-GISEL-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-FAKE16-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-FAKE16-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX11-FAKE16-GISEL-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-FAKE16-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-FAKE16-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-GISEL-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-FAKE16-GISEL-NEXT: s_endpgm
;
; GFX12-TRUE16-SDAG-LABEL: sqrt_v2f16:
; GFX12-TRUE16-SDAG: ; %bb.0: ; %entry
@@ -348,27 +367,45 @@ define amdgpu_kernel void @sqrt_v2f16(
; GFX12-TRUE16-GISEL-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX12-TRUE16-GISEL-NEXT: s_endpgm
;
-; GFX12-FAKE16-LABEL: sqrt_v2f16:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX12-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX12-FAKE16-NEXT: s_mov_b32 s11, s7
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX12-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-FAKE16-NEXT: v_sqrt_f16_e32 v0, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
-; GFX12-FAKE16-NEXT: v_sqrt_f16_e32 v1, v1
-; GFX12-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX12-FAKE16-NEXT: buffer_store_b32 v0, off, s[4:7], null
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-FAKE16-SDAG-LABEL: sqrt_v2f16:
+; GFX12-FAKE16-SDAG: ; %bb.0: ; %entry
+; GFX12-FAKE16-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-SDAG-NEXT: s_mov_b32 s6, -1
+; GFX12-FAKE16-SDAG-NEXT: s_mov_b32 s7, 0x31016000
+; GFX12-FAKE16-SDAG-NEXT: s_mov_b32 s10, s6
+; GFX12-FAKE16-SDAG-NEXT: s_mov_b32 s11, s7
+; GFX12-FAKE16-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-SDAG-NEXT: s_mov_b32 s8, s2
+; GFX12-FAKE16-SDAG-NEXT: s_mov_b32 s9, s3
+; GFX12-FAKE16-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX12-FAKE16-SDAG-NEXT: buffer_load_b32 v0, off, s[8:11], null
+; GFX12-FAKE16-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX12-FAKE16-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-FAKE16-SDAG-NEXT: v_sqrt_f16_e32 v0, v0
+; GFX12-FAKE16-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-FAKE16-SDAG-NEXT: v_sqrt_f16_e32 v1, v1
+; GFX12-FAKE16-SDAG-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX12-FAKE16-SDAG-NEXT: buffer_store_b32 v0, off, s[4:7], null
+; GFX12-FAKE16-SDAG-NEXT: s_endpgm
+;
+; GFX12-FAKE16-GISEL-LABEL: sqrt_v2f16:
+; GFX12-FAKE16-GISEL: ; %bb.0: ; %entry
+; GFX12-FAKE16-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-GISEL-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX12-FAKE16-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX12-FAKE16-GISEL-NEXT: v_s_sqrt_f16 s2, s2
+; GFX12-FAKE16-GISEL-NEXT: v_s_sqrt_f16 s3, s3
+; GFX12-FAKE16-GISEL-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-FAKE16-GISEL-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX12-FAKE16-GISEL-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-FAKE16-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-FAKE16-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX12-FAKE16-GISEL-NEXT: buffer_store_b32 v0, off, s[0:3], null
+; GFX12-FAKE16-GISEL-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) {
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix.ll b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
index bedf98c713831..07ac9c22d6c53 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
@@ -216,47 +216,89 @@ define <2 x float> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x hal
; GFX906-NEXT: v_mov_b32_e32 v0, v3
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_v2f32:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v4, v0
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v6, v1
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
-; GFX9GEN-NEXT: v_mac_f32_e32 v1, v3, v5
-; GFX9GEN-NEXT: v_mac_f32_e32 v0, v4, v6
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_v2f32:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v4, v0
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v6, v1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-GFX9GEN-NEXT: v_mac_f32_e32 v1, v3, v5
+; SDAG-GFX9GEN-NEXT: v_mac_f32_e32 v0, v4, v6
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v2f32:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v4, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v6, v1
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v2
-; VI-NEXT: v_mac_f32_e32 v1, v3, v5
-; VI-NEXT: v_mac_f32_e32 v0, v4, v6
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v2f32:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v4, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v6, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v1, v3, v5
+; SDAG-VI-NEXT: v_mac_f32_e32 v0, v4, v6
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_v2f32:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; CI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; CI-NEXT: v_cvt_f32_f16_e32 v6, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; CI-NEXT: v_cvt_f32_f16_e32 v7, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v3
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v2
-; CI-NEXT: v_mac_f32_e32 v1, v5, v4
-; CI-NEXT: v_mac_f32_e32 v0, v6, v7
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_v2f32:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v6, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v7, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v3
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-CI-NEXT: v_mac_f32_e32 v1, v5, v4
+; SDAG-CI-NEXT: v_mac_f32_e32 v0, v6, v7
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_v2f32:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v5, v1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_mac_f32_e32 v0, v3, v5
+; GISEL-GFX9GEN-NEXT: v_mac_f32_e32 v1, v4, v6
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_v2f32:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v5, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mac_f32_e32 v0, v3, v5
+; GISEL-VI-NEXT: v_mac_f32_e32 v1, v4, v6
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_v2f32:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v1
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v6, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_mac_f32_e32 v0, v4, v5
+; GISEL-CI-NEXT: v_mac_f32_e32 v1, v3, v6
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
%src2.ext = fpext <2 x half> %src2 to <2 x float>
@@ -316,20 +358,35 @@ define <2 x float> @v_mad_mix_v2f32_shuffle(<2 x half> %src0, <2 x half> %src1,
; VI-NEXT: v_mov_b32_e32 v1, v2
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_v2f32_shuffle:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; CI-NEXT: v_cvt_f32_f16_e32 v5, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v2
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v3
-; CI-NEXT: v_mad_f32 v0, v4, v0, v1
-; CI-NEXT: v_mac_f32_e32 v1, v5, v2
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_v2f32_shuffle:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v5, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v2, v3
+; SDAG-CI-NEXT: v_mad_f32 v0, v4, v0, v1
+; SDAG-CI-NEXT: v_mac_f32_e32 v1, v5, v2
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_v2f32_shuffle:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-CI-NEXT: v_mad_f32 v0, v3, v5, v1
+; GISEL-CI-NEXT: v_mac_f32_e32 v1, v4, v2
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.shuf = shufflevector <2 x half> %src0, <2 x half> poison, <2 x i32> <i32 1, i32 0>
%src1.shuf = shufflevector <2 x half> %src1, <2 x half> poison, <2 x i32> <i32 0, i32 1>
%src2.shuf = shufflevector <2 x half> %src2, <2 x half> poison, <2 x i32> <i32 1, i32 1>
@@ -1143,40 +1200,40 @@ define <2 x float> @v_mad_mix_v2f32_f32imm1(<2 x half> %src0, <2 x half> %src1)
; SDAG-GFX906-NEXT: v_mov_b32_e32 v0, v2
; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_v2f32_f32imm1:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v3, v1
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_mad_f32 v0, v0, v3, 1.0
-; GFX9GEN-NEXT: v_mad_f32 v1, v2, v1, 1.0
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_v2f32_f32imm1:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v3, v1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_mad_f32 v0, v0, v3, 1.0
+; SDAG-GFX9GEN-NEXT: v_mad_f32 v1, v2, v1, 1.0
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v2f32_f32imm1:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v1
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_mad_f32 v0, v0, v3, 1.0
-; VI-NEXT: v_mad_f32 v1, v2, v1, 1.0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v2f32_f32imm1:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_mad_f32 v0, v0, v3, 1.0
+; SDAG-VI-NEXT: v_mad_f32 v1, v2, v1, 1.0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_v2f32_f32imm1:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; CI-NEXT: v_mad_f32 v0, v0, v1, 1.0
-; CI-NEXT: v_mad_f32 v1, v3, v2, 1.0
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_v2f32_f32imm1:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-CI-NEXT: v_mad_f32 v0, v0, v1, 1.0
+; SDAG-CI-NEXT: v_mad_f32 v1, v3, v2, 1.0
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX1100-LABEL: v_mad_mix_v2f32_f32imm1:
; GISEL-GFX1100: ; %bb.0:
@@ -1205,6 +1262,41 @@ define <2 x float> @v_mad_mix_v2f32_f32imm1(<2 x half> %src0, <2 x half> %src1)
; GISEL-GFX906-NEXT: v_fma_mix_f32 v1, v0, v1, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0]
; GISEL-GFX906-NEXT: v_mov_b32_e32 v0, v2
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_v2f32_f32imm1:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_mad_f32 v0, v2, v0, 1.0
+; GISEL-GFX9GEN-NEXT: v_mad_f32 v1, v3, v1, 1.0
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_v2f32_f32imm1:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mad_f32 v0, v2, v0, 1.0
+; GISEL-VI-NEXT: v_mad_f32 v1, v3, v1, 1.0
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_v2f32_f32imm1:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_mad_f32 v0, v0, v1, 1.0
+; GISEL-CI-NEXT: v_mad_f32 v1, v2, v3, 1.0
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
%result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> <float 1.0, float 1.0>)
@@ -1240,43 +1332,43 @@ define <2 x float> @v_mad_mix_v2f32_cvtf16imminv2pi(<2 x half> %src0, <2 x half>
; SDAG-GFX906-NEXT: v_mov_b32_e32 v0, v2
; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v3, v1
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_mov_b32_e32 v1, 0x3e230000
-; GFX9GEN-NEXT: v_madak_f32 v0, v0, v3, 0x3e230000
-; GFX9GEN-NEXT: v_mac_f32_e32 v1, v2, v4
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v3, v1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_mov_b32_e32 v1, 0x3e230000
+; SDAG-GFX9GEN-NEXT: v_madak_f32 v0, v0, v3, 0x3e230000
+; SDAG-GFX9GEN-NEXT: v_mac_f32_e32 v1, v2, v4
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v1
-; VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_mov_b32_e32 v1, 0x3e230000
-; VI-NEXT: v_madak_f32 v0, v0, v3, 0x3e230000
-; VI-NEXT: v_mac_f32_e32 v1, v2, v4
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_mov_b32_e32 v1, 0x3e230000
+; SDAG-VI-NEXT: v_madak_f32 v0, v0, v3, 0x3e230000
+; SDAG-VI-NEXT: v_mac_f32_e32 v1, v2, v4
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v4, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; CI-NEXT: v_mov_b32_e32 v1, 0x3e230000
-; CI-NEXT: v_madak_f32 v0, v0, v4, 0x3e230000
-; CI-NEXT: v_mac_f32_e32 v1, v3, v2
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-CI-NEXT: v_mov_b32_e32 v1, 0x3e230000
+; SDAG-CI-NEXT: v_madak_f32 v0, v0, v4, 0x3e230000
+; SDAG-CI-NEXT: v_mac_f32_e32 v1, v3, v2
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX1100-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
; GISEL-GFX1100: ; %bb.0:
@@ -1305,6 +1397,44 @@ define <2 x float> @v_mad_mix_v2f32_cvtf16imminv2pi(<2 x half> %src0, <2 x half>
; GISEL-GFX906-NEXT: v_fma_mix_f32 v1, v0, v1, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0]
; GISEL-GFX906-NEXT: v_mov_b32_e32 v0, v2
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_mov_b32_e32 v1, 0x3e230000
+; GISEL-GFX9GEN-NEXT: v_madak_f32 v0, v2, v0, 0x3e230000
+; GISEL-GFX9GEN-NEXT: v_mac_f32_e32 v1, v3, v4
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mov_b32_e32 v1, 0x3e230000
+; GISEL-VI-NEXT: v_madak_f32 v0, v2, v0, 0x3e230000
+; GISEL-VI-NEXT: v_mac_f32_e32 v1, v3, v4
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_v2f32_cvtf16imminv2pi:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_mov_b32_e32 v1, 0x3e230000
+; GISEL-CI-NEXT: v_madak_f32 v0, v0, v4, 0x3e230000
+; GISEL-CI-NEXT: v_mac_f32_e32 v1, v2, v3
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
%src2 = fpext <2 x half> <half 0xH3118, half 0xH3118> to <2 x float>
@@ -1341,41 +1471,41 @@ define <2 x float> @v_mad_mix_v2f32_f32imminv2pi(<2 x half> %src0, <2 x half> %s
; SDAG-GFX906-NEXT: v_mov_b32_e32 v0, v2
; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_v2f32_f32imminv2pi:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v3, v1
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_mad_f32 v0, v0, v3, 0.15915494
-; GFX9GEN-NEXT: v_mad_f32 v1, v2, v1, 0.15915494
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_v2f32_f32imminv2pi:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v3, v1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_mad_f32 v0, v0, v3, 0.15915494
+; SDAG-GFX9GEN-NEXT: v_mad_f32 v1, v2, v1, 0.15915494
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v2f32_f32imminv2pi:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v1
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_mad_f32 v0, v0, v3, 0.15915494
-; VI-NEXT: v_mad_f32 v1, v2, v1, 0.15915494
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v2f32_f32imminv2pi:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_mad_f32 v0, v0, v3, 0.15915494
+; SDAG-VI-NEXT: v_mad_f32 v1, v2, v1, 0.15915494
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_v2f32_f32imminv2pi:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v4, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; CI-NEXT: v_mov_b32_e32 v1, 0x3e22f983
-; CI-NEXT: v_madak_f32 v0, v0, v4, 0x3e22f983
-; CI-NEXT: v_mac_f32_e32 v1, v3, v2
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_v2f32_f32imminv2pi:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-CI-NEXT: v_mov_b32_e32 v1, 0x3e22f983
+; SDAG-CI-NEXT: v_madak_f32 v0, v0, v4, 0x3e22f983
+; SDAG-CI-NEXT: v_mac_f32_e32 v1, v3, v2
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX1100-LABEL: v_mad_mix_v2f32_f32imminv2pi:
; GISEL-GFX1100: ; %bb.0:
@@ -1404,6 +1534,42 @@ define <2 x float> @v_mad_mix_v2f32_f32imminv2pi(<2 x half> %src0, <2 x half> %s
; GISEL-GFX906-NEXT: v_fma_mix_f32 v1, v0, v1, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0]
; GISEL-GFX906-NEXT: v_mov_b32_e32 v0, v2
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_v2f32_f32imminv2pi:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_mad_f32 v0, v2, v0, 0.15915494
+; GISEL-GFX9GEN-NEXT: v_mad_f32 v1, v3, v1, 0.15915494
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_v2f32_f32imminv2pi:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mad_f32 v0, v2, v0, 0.15915494
+; GISEL-VI-NEXT: v_mad_f32 v1, v3, v1, 0.15915494
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_v2f32_f32imminv2pi:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_mov_b32_e32 v1, 0x3e22f983
+; GISEL-CI-NEXT: v_madak_f32 v0, v0, v4, 0x3e22f983
+; GISEL-CI-NEXT: v_mac_f32_e32 v1, v2, v3
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
%src2 = fpext <2 x half> <half 0xH3118, half 0xH3118> to <2 x float>
@@ -2450,16 +2616,16 @@ define <2 x float> @v_mad_mix_v2f32_cvt_add(<2 x half> %src0, <2 x half> %src1)
; GFX1100-NEXT: v_mov_b32_e32 v0, v2
; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_v2f32_cvt_add:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v4, v1
-; GFX900-NEXT: v_add_f32_e32 v1, v2, v3
-; GFX900-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_v2f32_cvt_add:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v4, v1
+; SDAG-GFX900-NEXT: v_add_f32_e32 v1, v2, v3
+; SDAG-GFX900-NEXT: v_add_f32_e32 v0, v0, v4
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: v_mad_mix_v2f32_cvt_add:
; GFX906: ; %bb.0:
@@ -2469,40 +2635,86 @@ define <2 x float> @v_mad_mix_v2f32_cvt_add(<2 x half> %src0, <2 x half> %src1)
; GFX906-NEXT: v_mov_b32_e32 v0, v2
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_v2f32_cvt_add:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v4, v1
-; GFX9GEN-NEXT: v_add_f32_e32 v1, v2, v3
-; GFX9GEN-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_v2f32_cvt_add:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v4, v1
+; SDAG-GFX9GEN-NEXT: v_add_f32_e32 v1, v2, v3
+; SDAG-GFX9GEN-NEXT: v_add_f32_e32 v0, v0, v4
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v2f32_cvt_add:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v4, v1
-; VI-NEXT: v_add_f32_e32 v1, v2, v3
-; VI-NEXT: v_add_f32_e32 v0, v0, v4
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v2f32_cvt_add:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; SDAG-VI-NEXT: v_add_f32_e32 v1, v2, v3
+; SDAG-VI-NEXT: v_add_f32_e32 v0, v0, v4
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_v2f32_cvt_add:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; CI-NEXT: v_cvt_f32_f16_e32 v4, v1
-; CI-NEXT: v_add_f32_e32 v1, v3, v2
-; CI-NEXT: v_add_f32_e32 v0, v0, v4
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_v2f32_cvt_add:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; SDAG-CI-NEXT: v_add_f32_e32 v1, v3, v2
+; SDAG-CI-NEXT: v_add_f32_e32 v0, v0, v4
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX900-LABEL: v_mad_mix_v2f32_cvt_add:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v0, v1
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX900-NEXT: v_add_f32_e32 v0, v2, v0
+; GISEL-GFX900-NEXT: v_add_f32_e32 v1, v3, v1
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_v2f32_cvt_add:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_add_f32_e32 v0, v2, v0
+; GISEL-GFX9GEN-NEXT: v_add_f32_e32 v1, v3, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_v2f32_cvt_add:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_add_f32_e32 v0, v2, v0
+; GISEL-VI-NEXT: v_add_f32_e32 v1, v3, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_v2f32_cvt_add:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-CI-NEXT: v_add_f32_e32 v1, v2, v3
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
%result = fadd <2 x float> %src0.ext, %src1.ext
@@ -2560,18 +2772,31 @@ define <2 x float> @v_mad_mix_v2f32_shuffle_cvt_add(<2 x half> %src0, <2 x half>
; VI-NEXT: v_add_f32_e32 v1, v3, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_v2f32_shuffle_cvt_add:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; CI-NEXT: v_cvt_f32_f16_e32 v4, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v2
-; CI-NEXT: v_add_f32_e32 v0, v3, v0
-; CI-NEXT: v_add_f32_e32 v1, v4, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_v2f32_shuffle_cvt_add:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v4, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SDAG-CI-NEXT: v_add_f32_e32 v0, v3, v0
+; SDAG-CI-NEXT: v_add_f32_e32 v1, v4, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_v2f32_shuffle_cvt_add:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GISEL-CI-NEXT: v_add_f32_e32 v0, v2, v1
+; GISEL-CI-NEXT: v_add_f32_e32 v1, v3, v4
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.shuf = shufflevector <2 x half> %src0, <2 x half> poison, <2 x i32> <i32 1, i32 0>
%src1.shuf = shufflevector <2 x half> %src1, <2 x half> poison, <2 x i32> <i32 0, i32 1>
%src0.ext = fpext <2 x half> %src0.shuf to <2 x float>
@@ -3854,16 +4079,16 @@ define <2 x float> @v_mad_mix_v2f32_cvt_mul(<2 x half> %src0, <2 x half> %src1)
; GFX1100-NEXT: v_mov_b32_e32 v0, v2
; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_v2f32_cvt_mul:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v4, v1
-; GFX900-NEXT: v_mul_f32_e32 v1, v2, v3
-; GFX900-NEXT: v_mul_f32_e32 v0, v0, v4
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_v2f32_cvt_mul:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v4, v1
+; SDAG-GFX900-NEXT: v_mul_f32_e32 v1, v2, v3
+; SDAG-GFX900-NEXT: v_mul_f32_e32 v0, v0, v4
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: v_mad_mix_v2f32_cvt_mul:
; GFX906: ; %bb.0:
@@ -3873,40 +4098,86 @@ define <2 x float> @v_mad_mix_v2f32_cvt_mul(<2 x half> %src0, <2 x half> %src1)
; GFX906-NEXT: v_mov_b32_e32 v0, v2
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_v2f32_cvt_mul:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v4, v1
-; GFX9GEN-NEXT: v_mul_f32_e32 v1, v2, v3
-; GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v4
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_v2f32_cvt_mul:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v4, v1
+; SDAG-GFX9GEN-NEXT: v_mul_f32_e32 v1, v2, v3
+; SDAG-GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v4
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v2f32_cvt_mul:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v4, v1
-; VI-NEXT: v_mul_f32_e32 v1, v2, v3
-; VI-NEXT: v_mul_f32_e32 v0, v0, v4
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v2f32_cvt_mul:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; SDAG-VI-NEXT: v_mul_f32_e32 v1, v2, v3
+; SDAG-VI-NEXT: v_mul_f32_e32 v0, v0, v4
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_v2f32_cvt_mul:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; CI-NEXT: v_cvt_f32_f16_e32 v4, v1
-; CI-NEXT: v_mul_f32_e32 v1, v3, v2
-; CI-NEXT: v_mul_f32_e32 v0, v0, v4
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_v2f32_cvt_mul:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; SDAG-CI-NEXT: v_mul_f32_e32 v1, v3, v2
+; SDAG-CI-NEXT: v_mul_f32_e32 v0, v0, v4
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX900-LABEL: v_mad_mix_v2f32_cvt_mul:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v0, v1
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX900-NEXT: v_mul_f32_e32 v0, v2, v0
+; GISEL-GFX900-NEXT: v_mul_f32_e32 v1, v3, v1
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_v2f32_cvt_mul:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_mul_f32_e32 v0, v2, v0
+; GISEL-GFX9GEN-NEXT: v_mul_f32_e32 v1, v3, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_v2f32_cvt_mul:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mul_f32_e32 v0, v2, v0
+; GISEL-VI-NEXT: v_mul_f32_e32 v1, v3, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_v2f32_cvt_mul:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-CI-NEXT: v_mul_f32_e32 v1, v2, v3
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
%result = fmul <2 x float> %src0.ext, %src1.ext
@@ -3964,18 +4235,31 @@ define <2 x float> @v_mad_mix_v2f32_shuffle_cvt_mul(<2 x half> %src0, <2 x half>
; VI-NEXT: v_mul_f32_e32 v1, v3, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_v2f32_shuffle_cvt_mul:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; CI-NEXT: v_cvt_f32_f16_e32 v4, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v2
-; CI-NEXT: v_mul_f32_e32 v0, v3, v0
-; CI-NEXT: v_mul_f32_e32 v1, v4, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_v2f32_shuffle_cvt_mul:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v4, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SDAG-CI-NEXT: v_mul_f32_e32 v0, v3, v0
+; SDAG-CI-NEXT: v_mul_f32_e32 v1, v4, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_v2f32_shuffle_cvt_mul:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GISEL-CI-NEXT: v_mul_f32_e32 v0, v2, v1
+; GISEL-CI-NEXT: v_mul_f32_e32 v1, v3, v4
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.shuf = shufflevector <2 x half> %src0, <2 x half> poison, <2 x i32> <i32 1, i32 0>
%src1.shuf = shufflevector <2 x half> %src1, <2 x half> poison, <2 x i32> <i32 0, i32 1>
%src0.ext = fpext <2 x half> %src0.shuf to <2 x float>
diff --git a/llvm/test/CodeGen/AMDGPU/roundeven.ll b/llvm/test/CodeGen/AMDGPU/roundeven.ll
index 7230f2e8782dd..af6ef0790264c 100644
--- a/llvm/test/CodeGen/AMDGPU/roundeven.ll
+++ b/llvm/test/CodeGen/AMDGPU/roundeven.ll
@@ -466,17 +466,17 @@ define <2 x half> @v_roundeven_v2f16(<2 x half> %x) {
; GFX9-LABEL: v_roundeven_v2f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rndne_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rndne_f16_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-NEXT: v_rndne_f16_e32 v1, v0
+; GFX9-NEXT: v_rndne_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_roundeven_v2f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rndne_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rndne_f16_e32 v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-NEXT: v_rndne_f16_e32 v1, v0
+; GFX10-NEXT: v_rndne_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_roundeven_v2f16:
@@ -608,17 +608,19 @@ define <2 x half> @v_roundeven_v2f16_fneg(<2 x half> %x) {
; GFX9-LABEL: v_roundeven_v2f16_fneg:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rndne_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rndne_f16_e64 v0, -v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_rndne_f16_e32 v1, v0
+; GFX9-NEXT: v_rndne_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_roundeven_v2f16_fneg:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rndne_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rndne_f16_e64 v0, -v0
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX10-NEXT: v_rndne_f16_e32 v1, v0
+; GFX10-NEXT: v_rndne_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_roundeven_v2f16_fneg:
@@ -632,9 +634,10 @@ define <2 x half> @v_roundeven_v2f16_fneg(<2 x half> %x) {
; GFX11-FAKE16-LABEL: v_roundeven_v2f16_fneg:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: v_rndne_f16_e64 v0, -v0
-; GFX11-FAKE16-NEXT: v_rndne_f16_e64 v1, -v1
+; GFX11-FAKE16-NEXT: v_rndne_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_rndne_f16_e32 v1, v1
; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -773,23 +776,23 @@ define <4 x half> @v_roundeven_v4f16(<4 x half> %x) {
; GFX9-LABEL: v_roundeven_v4f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_rndne_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rndne_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_rndne_f16_e32 v1, v1
-; GFX9-NEXT: v_rndne_f16_e32 v0, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v3
-; GFX9-NEXT: v_pack_b32_f16 v1, v1, v2
+; GFX9-NEXT: v_rndne_f16_e32 v2, v0
+; GFX9-NEXT: v_rndne_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_rndne_f16_e32 v3, v1
+; GFX9-NEXT: v_rndne_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX9-NEXT: v_pack_b32_f16 v1, v3, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_roundeven_v4f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_rndne_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rndne_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_rndne_f16_e32 v0, v0
-; GFX10-NEXT: v_rndne_f16_e32 v1, v1
-; GFX10-NEXT: v_pack_b32_f16 v0, v0, v3
-; GFX10-NEXT: v_pack_b32_f16 v1, v1, v2
+; GFX10-NEXT: v_rndne_f16_e32 v2, v0
+; GFX10-NEXT: v_rndne_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_rndne_f16_e32 v3, v1
+; GFX10-NEXT: v_rndne_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_pack_b32_f16 v0, v2, v0
+; GFX10-NEXT: v_pack_b32_f16 v1, v3, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_roundeven_v4f16:
@@ -806,8 +809,8 @@ define <4 x half> @v_roundeven_v4f16(<4 x half> %x) {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-FAKE16-NEXT: v_rndne_f16_e32 v1, v1
; GFX11-FAKE16-NEXT: v_rndne_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT: v_rndne_f16_e32 v1, v1
; GFX11-FAKE16-NEXT: v_rndne_f16_e32 v2, v2
; GFX11-FAKE16-NEXT: v_rndne_f16_e32 v3, v3
; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
diff --git a/llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll
index c28e595521c49..25b2b0998e889 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll
@@ -111,18 +111,18 @@ define <2 x half> @test_ldexp_v2f16_v2i32(ptr addrspace(1) %out, <2 x half> %a,
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_ldexp_v2f16_v2i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x8000
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x7fff
-; GFX9-NEXT: v_med3_i32 v1, v3, s4, v0
-; GFX9-NEXT: v_med3_i32 v0, v4, s4, v0
-; GFX9-NEXT: v_ldexp_f16_e32 v1, v2, v1
-; GFX9-NEXT: v_ldexp_f16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_ldexp_v2f16_v2i32:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX9-SDAG-NEXT: v_med3_i32 v1, v3, s4, v0
+; GFX9-SDAG-NEXT: v_med3_i32 v0, v4, s4, v0
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v1, v2, v1
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_ldexp_v2f16_v2i32:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -150,6 +150,18 @@ define <2 x half> @test_ldexp_v2f16_v2i32(ptr addrspace(1) %out, <2 x half> %a,
; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-GISEL-LABEL: test_ldexp_v2f16_v2i32:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff8000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX9-GISEL-NEXT: v_med3_i32 v3, v3, v0, v1
+; GFX9-GISEL-NEXT: v_med3_i32 v0, v4, v0, v1
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v3, v2, v3
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_ldexp_v2f16_v2i32:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -165,15 +177,17 @@ define <2 x half> @test_ldexp_v2f16_v2i32(ptr addrspace(1) %out, <2 x half> %a,
; GFX11-GISEL-FAKE16-LABEL: test_ldexp_v2f16_v2i32:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v0, v3, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, v4, s0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, 0xffff8000, v3, v0
; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v3, v1
-; GFX11-GISEL-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v0, 0xffff8000, v4, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v2, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v3, v0
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x half> @llvm.experimental.constrained.ldexp.v2f16.v2i32(<2 x half> %a, <2 x i32> %b, metadata !"round.dynamic", metadata !"fpexcept.strict")
ret <2 x half> %result
@@ -194,20 +208,20 @@ define <3 x half> @test_ldexp_v3f16_v3i32(ptr addrspace(1) %out, <3 x half> %a,
; GFX8-NEXT: v_ldexp_f16_e32 v1, v3, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_ldexp_v3f16_v3i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x8000
-; GFX9-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX9-NEXT: v_med3_i32 v0, v4, s4, v1
-; GFX9-NEXT: v_med3_i32 v4, v5, s4, v1
-; GFX9-NEXT: v_ldexp_f16_e32 v0, v2, v0
-; GFX9-NEXT: v_ldexp_f16_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b32 s5, 0x5040100
-; GFX9-NEXT: v_med3_i32 v1, v6, s4, v1
-; GFX9-NEXT: v_perm_b32 v0, v2, v0, s5
-; GFX9-NEXT: v_ldexp_f16_e32 v1, v3, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_ldexp_v3f16_v3i32:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX9-SDAG-NEXT: v_med3_i32 v0, v4, s4, v1
+; GFX9-SDAG-NEXT: v_med3_i32 v4, v5, s4, v1
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-SDAG-NEXT: s_mov_b32 s5, 0x5040100
+; GFX9-SDAG-NEXT: v_med3_i32 v1, v6, s4, v1
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v2, v0, s5
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v1, v3, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_ldexp_v3f16_v3i32:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -240,6 +254,21 @@ define <3 x half> @test_ldexp_v3f16_v3i32(ptr addrspace(1) %out, <3 x half> %a,
; GFX11-SDAG-FAKE16-NEXT: v_ldexp_f16_e32 v1, v3, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-GISEL-LABEL: test_ldexp_v3f16_v3i32:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff8000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX9-GISEL-NEXT: v_med3_i32 v4, v4, v0, v1
+; GFX9-GISEL-NEXT: v_med3_i32 v5, v5, v0, v1
+; GFX9-GISEL-NEXT: v_med3_i32 v0, v6, v0, v1
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v4, v2, v4
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v1, v3, v0
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v2, 16, v4
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_ldexp_v3f16_v3i32:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -257,18 +286,22 @@ define <3 x half> @test_ldexp_v3f16_v3i32(ptr addrspace(1) %out, <3 x half> %a,
; GFX11-GISEL-FAKE16-LABEL: test_ldexp_v3f16_v3i32:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v0, v4, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, v5, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v0
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v2, v6, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v4, v1
-; GFX11-GISEL-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v3, v2
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, 0xffff8000, v4, v0
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v4, 0xffff8000, v6, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v0, 0xffff8000, v5, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v2, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v2, v3, v4
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v6, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, s0, 16, v2
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <3 x half> @llvm.experimental.constrained.ldexp.v3f16.v3i32(<3 x half> %a, <3 x i32> %b, metadata !"round.dynamic", metadata !"fpexcept.strict")
ret <3 x half> %result
@@ -292,23 +325,23 @@ define <4 x half> @test_ldexp_v4f16_v4i32(ptr addrspace(1) %out, <4 x half> %a,
; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_ldexp_v4f16_v4i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0x8000
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x7fff
-; GFX9-NEXT: v_med3_i32 v1, v6, s4, v0
-; GFX9-NEXT: v_med3_i32 v6, v7, s4, v0
-; GFX9-NEXT: v_med3_i32 v4, v4, s4, v0
-; GFX9-NEXT: v_med3_i32 v0, v5, s4, v0
-; GFX9-NEXT: v_ldexp_f16_e32 v1, v3, v1
-; GFX9-NEXT: v_ldexp_f16_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_ldexp_f16_e32 v4, v2, v4
-; GFX9-NEXT: v_ldexp_f16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: v_perm_b32 v0, v0, v4, s4
-; GFX9-NEXT: v_perm_b32 v1, v3, v1, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_ldexp_v4f16_v4i32:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX9-SDAG-NEXT: v_med3_i32 v1, v6, s4, v0
+; GFX9-SDAG-NEXT: v_med3_i32 v6, v7, s4, v0
+; GFX9-SDAG-NEXT: v_med3_i32 v4, v4, s4, v0
+; GFX9-SDAG-NEXT: v_med3_i32 v0, v5, s4, v0
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v1, v3, v1
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v4, v2, v4
+; GFX9-SDAG-NEXT: v_ldexp_f16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v0, v4, s4
+; GFX9-SDAG-NEXT: v_perm_b32 v1, v3, v1, s4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_ldexp_v4f16_v4i32:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -348,6 +381,23 @@ define <4 x half> @test_ldexp_v4f16_v4i32(ptr addrspace(1) %out, <4 x half> %a,
; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v1, v1, v3, 0x5040100
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-GISEL-LABEL: test_ldexp_v4f16_v4i32:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff8000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX9-GISEL-NEXT: v_med3_i32 v4, v4, v0, v1
+; GFX9-GISEL-NEXT: v_med3_i32 v5, v5, v0, v1
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v4, v2, v4
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_med3_i32 v5, v6, v0, v1
+; GFX9-GISEL-NEXT: v_med3_i32 v0, v7, v0, v1
+; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v5, v3, v5
+; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v2, 16, v4
+; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v1, 16, v5
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-GISEL-TRUE16-LABEL: test_ldexp_v4f16_v4i32:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -368,22 +418,26 @@ define <4 x half> @test_ldexp_v4f16_v4i32(ptr addrspace(1) %out, <4 x half> %a,
; GFX11-GISEL-FAKE16-LABEL: test_ldexp_v4f16_v4i32:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v0, v6, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, v7, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v4, v4, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v5, v5, s0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v3, v3, v0
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v2, v4
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v4, 0xffff8000, v4, v0
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v6, 0xffff8000, v6, v0
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v5, 0xffff8000, v5, v0
+; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v0, 0xffff8000, v7, v0
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v2, v2, v4
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v2, v6, v5
-; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v7, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x5040100
-; GFX11-GISEL-FAKE16-NEXT: v_perm_b32 v1, v1, v3, 0x5040100
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v3, v3, v6
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v1, v1, v5
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v4, v8, v0
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v2
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x half> @llvm.experimental.constrained.ldexp.v4f16.v4i32(<4 x half> %a, <4 x i32> %b, metadata !"round.dynamic", metadata !"fpexcept.strict")
ret <4 x half> %result
@@ -471,3 +525,4 @@ attributes #1 = { nocallback nofree nosync nounwind willreturn memory(inaccessib
; GFX11: {{.*}}
; GFX11-GISEL: {{.*}}
; GFX11-SDAG: {{.*}}
+; GFX9: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
index dd6acd5d68892..23e2af2134c22 100644
--- a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
@@ -1576,23 +1576,24 @@ define i16 @basic_smax_smin_vec_input(<2 x i16> %src) {
; GISEL-VI: ; %bb.0:
; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-VI-NEXT: v_mov_b32_e32 v1, 0xff
-; GISEL-VI-NEXT: v_min_i16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GISEL-VI-NEXT: v_min_i16_e32 v0, 0xff, v0
-; GISEL-VI-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-VI-NEXT: v_min_i16_e32 v2, 0xff, v0
+; GISEL-VI-NEXT: v_min_i16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GISEL-VI-NEXT: v_max_i16_e32 v0, 0, v0
-; GISEL-VI-NEXT: v_max_i16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GISEL-VI-NEXT: v_or_b32_e32 v0, v0, v1
+; GISEL-VI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GISEL-VI-NEXT: v_max_i16_e32 v1, 0, v2
+; GISEL-VI-NEXT: v_lshlrev_b16_e32 v0, 8, v0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX9-LABEL: basic_smax_smin_vec_input:
; GISEL-GFX9: ; %bb.0:
; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX9-NEXT: s_movk_i32 s4, 0xff
-; GISEL-GFX9-NEXT: v_pk_min_i16 v0, v0, s4 op_sel_hi:[1,0]
-; GISEL-GFX9-NEXT: v_pk_max_i16 v0, v0, 0
-; GISEL-GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GISEL-GFX9-NEXT: v_or_b32_e32 v0, v0, v1
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0xff00ff
+; GISEL-GFX9-NEXT: v_pk_min_i16 v0, v1, v0
+; GISEL-GFX9-NEXT: v_pk_max_i16 v0, 0, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0xff
+; GISEL-GFX9-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GISEL-GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX11-TRUE16-LABEL: basic_smax_smin_vec_input:
@@ -1612,12 +1613,15 @@ define i16 @basic_smax_smin_vec_input(<2 x i16> %src) {
; GISEL-GFX11-FAKE16-LABEL: basic_smax_smin_vec_input:
; GISEL-GFX11-FAKE16: ; %bb.0:
; GISEL-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-FAKE16-NEXT: v_pk_min_i16 v0, 0xff, v0 op_sel_hi:[0,1]
+; GISEL-GFX11-FAKE16-NEXT: v_pk_min_i16 v0, 0xff00ff, v0
; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX11-FAKE16-NEXT: v_pk_max_i16 v0, v0, 0
+; GISEL-GFX11-FAKE16-NEXT: v_pk_max_i16 v0, 0, v0
; GISEL-GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GISEL-GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
+; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GISEL-GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GISEL-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1646,12 +1650,15 @@ define i16 @basic_smax_smin_vec_input(<2 x i16> %src) {
; GISEL-GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GISEL-GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GISEL-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GISEL-GFX12-FAKE16-NEXT: v_pk_min_i16 v0, 0xff, v0 op_sel_hi:[0,1]
+; GISEL-GFX12-FAKE16-NEXT: v_pk_min_i16 v0, 0xff00ff, v0
; GISEL-GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX12-FAKE16-NEXT: v_pk_max_i16 v0, v0, 0
+; GISEL-GFX12-FAKE16-NEXT: v_pk_max_i16 v0, 0, v0
; GISEL-GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GISEL-GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GISEL-GFX12-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
+; GISEL-GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GISEL-GFX12-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GISEL-GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1742,24 +1749,24 @@ define i16 @basic_smax_smin_vec_input_rev(<2 x i16> %src) {
; GISEL-VI-LABEL: basic_smax_smin_vec_input_rev:
; GISEL-VI: ; %bb.0:
; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-VI-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-VI-NEXT: v_max_i16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GISEL-VI-NEXT: v_max_i16_e32 v0, 0, v0
+; GISEL-VI-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-VI-NEXT: v_max_i16_e32 v1, 0, v0
+; GISEL-VI-NEXT: v_max_i16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GISEL-VI-NEXT: v_mov_b32_e32 v2, 0xff
-; GISEL-VI-NEXT: v_min_i16_e32 v0, 0xff, v0
-; GISEL-VI-NEXT: v_min_i16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GISEL-VI-NEXT: v_or_b32_e32 v0, v0, v1
+; GISEL-VI-NEXT: v_min_i16_e32 v1, 0xff, v1
+; GISEL-VI-NEXT: v_min_i16_sdwa v0, v0, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GISEL-VI-NEXT: v_or_b32_e32 v0, v1, v0
; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX9-LABEL: basic_smax_smin_vec_input_rev:
; GISEL-GFX9: ; %bb.0:
; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX9-NEXT: v_pk_max_i16 v0, v0, 0
-; GISEL-GFX9-NEXT: s_movk_i32 s4, 0xff
-; GISEL-GFX9-NEXT: v_pk_min_i16 v0, v0, s4 op_sel_hi:[1,0]
-; GISEL-GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GISEL-GFX9-NEXT: v_or_b32_e32 v0, v0, v1
+; GISEL-GFX9-NEXT: v_pk_max_i16 v0, 0, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0xff00ff
+; GISEL-GFX9-NEXT: v_pk_min_i16 v0, v1, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0xff
+; GISEL-GFX9-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GISEL-GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX11-TRUE16-LABEL: basic_smax_smin_vec_input_rev:
@@ -1776,12 +1783,15 @@ define i16 @basic_smax_smin_vec_input_rev(<2 x i16> %src) {
; GISEL-GFX11-FAKE16-LABEL: basic_smax_smin_vec_input_rev:
; GISEL-GFX11-FAKE16: ; %bb.0:
; GISEL-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-FAKE16-NEXT: v_pk_max_i16 v0, v0, 0
+; GISEL-GFX11-FAKE16-NEXT: v_pk_max_i16 v0, 0, v0
; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX11-FAKE16-NEXT: v_pk_min_i16 v0, 0xff, v0 op_sel_hi:[0,1]
+; GISEL-GFX11-FAKE16-NEXT: v_pk_min_i16 v0, 0xff00ff, v0
; GISEL-GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GISEL-GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
+; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GISEL-GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GISEL-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1807,12 +1817,15 @@ define i16 @basic_smax_smin_vec_input_rev(<2 x i16> %src) {
; GISEL-GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GISEL-GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GISEL-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GISEL-GFX12-FAKE16-NEXT: v_pk_max_i16 v0, v0, 0
+; GISEL-GFX12-FAKE16-NEXT: v_pk_max_i16 v0, 0, v0
; GISEL-GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX12-FAKE16-NEXT: v_pk_min_i16 v0, 0xff, v0 op_sel_hi:[0,1]
+; GISEL-GFX12-FAKE16-NEXT: v_pk_min_i16 v0, 0xff00ff, v0
; GISEL-GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GISEL-GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GISEL-GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GISEL-GFX12-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
+; GISEL-GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GISEL-GFX12-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GISEL-GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
index 52b3398c5fa81..eaad2245b257f 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
@@ -1,20 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define i8 @test_vector_reduce_add_v2i8(<2 x i8> %v) {
; GFX7-LABEL: test_vector_reduce_add_v2i8:
@@ -1153,12 +1153,20 @@ entry:
}
define i16 @test_vector_reduce_add_v2i16(<2 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_add_v2i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_add_v2i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_add_v2i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: test_vector_reduce_add_v2i16:
; GFX8: ; %bb.0: ; %entry
@@ -1298,21 +1306,36 @@ define i16 @test_vector_reduce_add_v3i16(<3 x i16> %v) {
; GFX8-NEXT: v_add_u16_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_add_v3i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_add_u16 v1, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_add_u16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_add_v3i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v0, v1
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_add_u16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_add_v3i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX10-NEXT: v_pk_add_u16 v0, v0, v1
-; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_add_v3i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_add_u16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_add_v3i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_add_v3i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-GISEL-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-GISEL-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_add_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1343,9 +1366,9 @@ define i16 @test_vector_reduce_add_v3i16(<3 x i16> %v) {
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_pk_add_u16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_add_nc_u16 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_add_v3i16:
@@ -1393,9 +1416,9 @@ define i16 @test_vector_reduce_add_v3i16(<3 x i16> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX12-GISEL-FAKE16-NEXT: v_pk_add_u16 v0, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-FAKE16-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX12-GISEL-FAKE16-NEXT: v_add_nc_u16 v0, v0, v1
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.add.v3i16(<3 x i16> %v)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
index 497d0e51b8742..7ffc8ad6add69 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
@@ -4,18 +4,18 @@
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; FIXME-TRUE16. enable gisel
; XUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; XUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
; FIXME: codegen regression, related to:
; - looking through sgpr to vgpr copy
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
index 31dc3a65dd8a3..e208e687a476c 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
@@ -232,10 +232,10 @@ define half @test_vector_reduce_fadd_v3half(half %sp, <3 x half> %v) {
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fadd_v3half:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v3
; GFX11-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v2
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -286,10 +286,10 @@ define half @test_vector_reduce_fadd_v3half(half %sp, <3 x half> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX12-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v3
; GFX12-GISEL-FAKE16-NEXT: v_add_f16_e32 v0, v0, v2
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmax.ll
index af05ba16ab7e0..fe85aa746f225 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmax.ll
@@ -1,59 +1,93 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1170 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1170 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1170 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1170 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1170 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define half @test_vector_reduce_fmax_v2half(<2 x half> %v) {
-; GFX7-LABEL: test_vector_reduce_fmax_v2half:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_max_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_fmax_v2half:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-SDAG-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_fmax_v2half:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-GISEL-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_fmax_v2half:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_fmax_v2half:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_fmax_v2half:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX9-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_fmax_v2half:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX10-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_fmax_v2half:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e32 v1, v0, v0
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_fmax_v2half:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX9-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_fmax_v2half:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_max_f16_e32 v1, v0, v0
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v0, v1, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_fmax_v2half:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX10-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_fmax_v2half:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_max_f16_e32 v1, v0, v0
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v0, v1, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmax_v2half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -204,38 +238,66 @@ define half @test_vector_reduce_fmax_v3half(<3 x half> %v) {
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_fmax_v3half:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v2
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_fmax_v3half:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX9-NEXT: s_movk_i32 s0, 0x7e00
-; GFX9-NEXT: v_pack_b32_f16 v1, v1, s0
-; GFX9-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_pk_max_f16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_fmax_v3half:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v1, v1, 0x7e00
-; GFX10-NEXT: v_pk_max_f16 v0, v0, v1
-; GFX10-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_fmax_v3half:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_fmax_v3half:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e32 v2, v0, v0
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_fmax_v3half:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX9-SDAG-NEXT: s_movk_i32 s0, 0x7e00
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v1, v1, s0
+; GFX9-SDAG-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_pk_max_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_fmax_v3half:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_max_f16_e32 v2, v0, v0
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX9-GISEL-NEXT: v_max3_f16 v0, v2, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_fmax_v3half:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX10-SDAG-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX10-SDAG-NEXT: v_pack_b32_f16 v1, v1, 0x7e00
+; GFX10-SDAG-NEXT: v_pk_max_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_fmax_v3half:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_max_f16_e32 v2, v0, v0
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX10-GISEL-NEXT: v_max3_f16 v0, v2, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmax_v3half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -274,14 +336,12 @@ define half @test_vector_reduce_fmax_v3half(<3 x half> %v) {
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmax_v3half:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, 0x7e00
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v0, v0, v2, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-SDAG-TRUE16-LABEL: test_vector_reduce_fmax_v3half:
@@ -321,14 +381,12 @@ define half @test_vector_reduce_fmax_v3half(<3 x half> %v) {
; GFX1170-GISEL-FAKE16-LABEL: test_vector_reduce_fmax_v3half:
; GFX1170-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v0
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, 0x7e00
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v1
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v1
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v2, v1
; GFX1170-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_fmax_v3half:
@@ -384,14 +442,12 @@ define half @test_vector_reduce_fmax_v3half(<3 x half> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v0
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, 0x7e00
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v2, v1
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call half @llvm.vector.reduce.fmax.v3half(<3 x half> %v)
@@ -437,37 +493,71 @@ define half @test_vector_reduce_fmax_v4half(<4 x half> %v) {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_fmax_v4half:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v3, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX8-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v3
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_fmax_v4half:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_max_f16 v1, v1, v1
-; GFX9-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_pk_max_f16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_fmax_v4half:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v1, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX10-NEXT: v_pk_max_f16 v0, v0, v1
-; GFX10-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_fmax_v4half:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v3, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v3
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_fmax_v4half:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e32 v2, v0, v0
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_max_f16_e32 v2, v1, v1
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v1, v2, v1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_fmax_v4half:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_pk_max_f16 v1, v1, v1
+; GFX9-SDAG-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_pk_max_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_fmax_v4half:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_max_f16_e32 v3, v1, v1
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v2, v0, v0
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v1, v3, v1
+; GFX9-GISEL-NEXT: v_max3_f16 v0, v2, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_fmax_v4half:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_pk_max_f16 v1, v1, v1
+; GFX10-SDAG-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX10-SDAG-NEXT: v_pk_max_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_fmax_v4half:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_max_f16_e32 v2, v1, v1
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v3, v0, v0
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v1, v2, v1
+; GFX10-GISEL-NEXT: v_max3_f16 v0, v3, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmax_v4half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -506,13 +596,16 @@ define half @test_vector_reduce_fmax_v4half(<4 x half> %v) {
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmax_v4half:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v1, v1, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v1, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v0, v0, v3, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-SDAG-TRUE16-LABEL: test_vector_reduce_fmax_v4half:
@@ -552,13 +645,16 @@ define half @test_vector_reduce_fmax_v4half(<4 x half> %v) {
; GFX1170-GISEL-FAKE16-LABEL: test_vector_reduce_fmax_v4half:
; GFX1170-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v1, v1, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v0
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v3, v1
; GFX1170-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_fmax_v4half:
@@ -614,13 +710,16 @@ define half @test_vector_reduce_fmax_v4half(<4 x half> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v1, v1, v1
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v0
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v3, v1
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call half @llvm.vector.reduce.fmax.v4half(<4 x half> %v)
@@ -702,53 +801,109 @@ define half @test_vector_reduce_fmax_v8half(<8 x half> %v) {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_fmax_v8half:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX8-NEXT: v_max_f16_sdwa v4, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v5
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v4
-; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v3, v3
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_fmax_v8half:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_max_f16 v3, v3, v3
-; GFX9-NEXT: v_pk_max_f16 v1, v1, v1
-; GFX9-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX9-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX9-NEXT: v_pk_max_f16 v1, v1, v3
-; GFX9-NEXT: v_pk_max_f16 v0, v0, v2
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_pk_max_f16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_fmax_v8half:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v3, v3, v3
-; GFX10-NEXT: v_pk_max_f16 v1, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX10-NEXT: v_pk_max_f16 v1, v1, v3
-; GFX10-NEXT: v_pk_max_f16 v0, v0, v2
-; GFX10-NEXT: v_pk_max_f16 v0, v0, v1
-; GFX10-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_fmax_v8half:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v5, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v4, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v5
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v2, v2
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v3, v3
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_fmax_v8half:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e32 v4, v0, v0
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v4, v0
+; GFX8-GISEL-NEXT: v_max_f16_e32 v4, v1, v1
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v1, v4, v1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v4, v2, v2
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v2, v4, v2
+; GFX8-GISEL-NEXT: v_max_f16_e32 v4, v3, v3
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v3, v4, v3
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v1, v2, v3
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_fmax_v8half:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_pk_max_f16 v3, v3, v3
+; GFX9-SDAG-NEXT: v_pk_max_f16 v1, v1, v1
+; GFX9-SDAG-NEXT: v_pk_max_f16 v2, v2, v2
+; GFX9-SDAG-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX9-SDAG-NEXT: v_pk_max_f16 v1, v1, v3
+; GFX9-SDAG-NEXT: v_pk_max_f16 v0, v0, v2
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_pk_max_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_fmax_v8half:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_max_f16_e32 v5, v1, v1
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v4, v0, v0
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v1, v5, v1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v5, v2, v2
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v3, v6, v3
+; GFX9-GISEL-NEXT: v_max3_f16 v0, v4, v0, v1
+; GFX9-GISEL-NEXT: v_max3_f16 v1, v5, v2, v3
+; GFX9-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_fmax_v8half:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_pk_max_f16 v3, v3, v3
+; GFX10-SDAG-NEXT: v_pk_max_f16 v1, v1, v1
+; GFX10-SDAG-NEXT: v_pk_max_f16 v2, v2, v2
+; GFX10-SDAG-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX10-SDAG-NEXT: v_pk_max_f16 v1, v1, v3
+; GFX10-SDAG-NEXT: v_pk_max_f16 v0, v0, v2
+; GFX10-SDAG-NEXT: v_pk_max_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_fmax_v8half:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_max_f16_e32 v4, v1, v1
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v5, v0, v0
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v1, v4, v1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v4, v2, v2
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v3, v6, v3
+; GFX10-GISEL-NEXT: v_max3_f16 v0, v5, v0, v1
+; GFX10-GISEL-NEXT: v_max3_f16 v1, v4, v2, v3
+; GFX10-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmax_v8half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -805,16 +960,23 @@ define half @test_vector_reduce_fmax_v8half(<8 x half> %v) {
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmax_v8half:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v3, v3, v3
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v1, v1, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v7, v7, v7
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v1, v1, v5
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v3, v3, v7
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v1, v1, v3
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v0, v0, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v0, v0, v4, v1
+; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v1, v2, v5, v3
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -874,16 +1036,23 @@ define half @test_vector_reduce_fmax_v8half(<8 x half> %v) {
; GFX1170-GISEL-FAKE16-LABEL: test_vector_reduce_fmax_v8half:
; GFX1170-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v3, v3, v3
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v1, v1, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v2, v2, v2
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v7, v7, v7
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v5
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v6, v6
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v7
; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v1, v1, v3
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v2
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v4, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v1, v2, v5, v3
; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v1
; GFX1170-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -959,16 +1128,23 @@ define half @test_vector_reduce_fmax_v8half(<8 x half> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v3, v3, v3
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v1, v1, v1
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v2, v2, v2
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v7, v7, v7
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v5
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v6, v6
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v7
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v1, v1, v3
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v2
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v4, v1
+; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v1, v2, v5, v3
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v1
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1124,85 +1300,183 @@ define half @test_vector_reduce_fmax_v16half(<16 x half> %v) {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_fmax_v16half:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v9, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX8-NEXT: v_max_f16_sdwa v8, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v9
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v8
-; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v3, v3
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v4, v4
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v5, v5
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v6, v6
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v7, v7
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_fmax_v16half:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_max_f16_sdwa v15, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v14, 16, v1
-; GFX9-NEXT: v_max_f16_e32 v0, v0, v15
-; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v2
-; GFX9-NEXT: v_max3_f16 v0, v0, v1, v14
-; GFX9-NEXT: v_lshrrev_b32_e32 v12, 16, v3
-; GFX9-NEXT: v_max3_f16 v0, v0, v2, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v4
-; GFX9-NEXT: v_max3_f16 v0, v0, v3, v12
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v5
-; GFX9-NEXT: v_max3_f16 v0, v0, v4, v11
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v6
-; GFX9-NEXT: v_max3_f16 v0, v0, v5, v10
-; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; GFX9-NEXT: v_max3_f16 v0, v0, v6, v9
-; GFX9-NEXT: v_max3_f16 v0, v0, v7, v8
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_fmax_v16half:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_max_f16_sdwa v8, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; GFX10-NEXT: v_max_f16_e32 v0, v0, v8
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX10-NEXT: v_max3_f16 v0, v0, v1, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX10-NEXT: v_max3_f16 v0, v0, v2, v8
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX10-NEXT: v_max3_f16 v0, v0, v3, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; GFX10-NEXT: v_max3_f16 v0, v0, v4, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; GFX10-NEXT: v_max3_f16 v0, v0, v5, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX10-NEXT: v_max3_f16 v0, v0, v6, v2
-; GFX10-NEXT: v_max3_f16 v0, v0, v7, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_fmax_v16half:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v9, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v8, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v9
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v2, v2
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v3, v3
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v4, v4
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v5, v5
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v6, v6
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v7, v7
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_fmax_v16half:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v0, v0
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v8, v0
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v1, v1
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v1, v8, v1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v2, v2
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v2, v8, v2
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v3, v3
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v3, v8, v3
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v4, v4
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v4, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v4, v8, v4
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v5, v5
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v5, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v5, v8, v5
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v6, v6
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v6, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v6, v8, v6
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v7, v7
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v7, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v7, v8, v7
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v1, v2, v3
+; GFX8-GISEL-NEXT: v_max_f16_e32 v2, v4, v5
+; GFX8-GISEL-NEXT: v_max_f16_e32 v3, v6, v7
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v1, v2, v3
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_fmax_v16half:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_max_f16_sdwa v15, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v14, 16, v1
+; GFX9-SDAG-NEXT: v_max_f16_e32 v0, v0, v15
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v13, 16, v2
+; GFX9-SDAG-NEXT: v_max3_f16 v0, v0, v1, v14
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v12, 16, v3
+; GFX9-SDAG-NEXT: v_max3_f16 v0, v0, v2, v13
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v11, 16, v4
+; GFX9-SDAG-NEXT: v_max3_f16 v0, v0, v3, v12
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX9-SDAG-NEXT: v_max3_f16 v0, v0, v4, v11
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v9, 16, v6
+; GFX9-SDAG-NEXT: v_max3_f16 v0, v0, v5, v10
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-SDAG-NEXT: v_max3_f16 v0, v0, v6, v9
+; GFX9-SDAG-NEXT: v_max3_f16 v0, v0, v7, v8
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_fmax_v16half:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_max_f16_e32 v9, v1, v1
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v10, v3, v3
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v11, v5, v5
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v5, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v12, v7, v7
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v7, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v8, v0, v0
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v1, v9, v1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v9, v2, v2
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v3, v10, v3
+; GFX9-GISEL-NEXT: v_max_f16_e32 v10, v4, v4
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v4, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v5, v11, v5
+; GFX9-GISEL-NEXT: v_max_f16_e32 v11, v6, v6
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v6, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v7, v12, v7
+; GFX9-GISEL-NEXT: v_max3_f16 v0, v8, v0, v1
+; GFX9-GISEL-NEXT: v_max3_f16 v1, v9, v2, v3
+; GFX9-GISEL-NEXT: v_max3_f16 v2, v10, v4, v5
+; GFX9-GISEL-NEXT: v_max3_f16 v3, v11, v6, v7
+; GFX9-GISEL-NEXT: v_max_f16_e32 v2, v2, v3
+; GFX9-GISEL-NEXT: v_max3_f16 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_fmax_v16half:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_max_f16_sdwa v8, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX10-SDAG-NEXT: v_max_f16_e32 v0, v0, v8
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX10-SDAG-NEXT: v_max3_f16 v0, v0, v1, v9
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; GFX10-SDAG-NEXT: v_max3_f16 v0, v0, v2, v8
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; GFX10-SDAG-NEXT: v_max3_f16 v0, v0, v3, v1
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; GFX10-SDAG-NEXT: v_max3_f16 v0, v0, v4, v2
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX10-SDAG-NEXT: v_max3_f16 v0, v0, v5, v1
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX10-SDAG-NEXT: v_max3_f16 v0, v0, v6, v2
+; GFX10-SDAG-NEXT: v_max3_f16 v0, v0, v7, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_fmax_v16half:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_max_f16_e32 v8, v1, v1
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v10, v5, v5
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v5, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v12, v7, v7
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v7, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v1, v8, v1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v8, v3, v3
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v11, v4, v4
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v4, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v5, v10, v5
+; GFX10-GISEL-NEXT: v_max_f16_e32 v10, v6, v6
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v6, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v7, v12, v7
+; GFX10-GISEL-NEXT: v_max_f16_e32 v9, v0, v0
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v12, v2, v2
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v3, v8, v3
+; GFX10-GISEL-NEXT: v_max3_f16 v4, v11, v4, v5
+; GFX10-GISEL-NEXT: v_max3_f16 v5, v10, v6, v7
+; GFX10-GISEL-NEXT: v_max3_f16 v0, v9, v0, v1
+; GFX10-GISEL-NEXT: v_max3_f16 v1, v12, v2, v3
+; GFX10-GISEL-NEXT: v_max_f16_e32 v2, v4, v5
+; GFX10-GISEL-NEXT: v_max3_f16 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmax_v16half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1287,29 +1561,42 @@ define half @test_vector_reduce_fmax_v16half(<16 x half> %v) {
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmax_v16half:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v9, v9, v9
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v7, v7, v7
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v1, v1, v9
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v9, v11, v11
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v11, v13, v13
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v13, v15, v15
; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v12, v12, v12
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v5, v5, v11
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v6, v6, v6
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v11, v14, v14
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v7, v7, v13
; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v8, v8, v8
-; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v0, v0, v8
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v0, v0, v1, v9
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v0, v0, v2, v8
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v0, v0, v3, v1
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v0, v0, v4, v2
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v0, v0, v5, v1
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v0, v0, v6, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v10, v10, v10
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v3, v3, v9
+; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v4, v4, v12, v5
+; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v5, v6, v11, v7
+; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v0, v0, v8, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v1, v2, v10, v3
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v2, v4, v5
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v0, v0, v7, v1
+; GFX11-GISEL-FAKE16-NEXT: v_max3_f16 v0, v0, v1, v2
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-SDAG-TRUE16-LABEL: test_vector_reduce_fmax_v16half:
@@ -1395,29 +1682,42 @@ define half @test_vector_reduce_fmax_v16half(<16 x half> %v) {
; GFX1170-GISEL-FAKE16-LABEL: test_vector_reduce_fmax_v16half:
; GFX1170-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v9, v9, v9
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v7, v7, v7
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v9
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v9, v11, v11
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v11, v13, v13
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v13, v15, v15
; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v12, v12, v12
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v11
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v6, v6, v6
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v11, v14, v14
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v7, v7, v13
; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v8, v8, v8
-; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v8
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v1, v9
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v2, v8
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v3, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v4, v2
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v5, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v6, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v10, v10, v10
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v9
+; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v4, v4, v12, v5
+; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v5, v6, v11, v7
+; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v8, v1
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v1, v2, v10, v3
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v4, v5
; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v7, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v1, v2
; GFX1170-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_fmax_v16half:
@@ -1519,29 +1819,42 @@ define half @test_vector_reduce_fmax_v16half(<16 x half> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v9, v9, v9
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v7, v7, v7
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v9
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v9, v11, v11
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v11, v13, v13
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v13, v15, v15
; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v12, v12, v12
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v11
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v6, v6, v6
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v11, v14, v14
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v7, v7, v13
; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v8, v8, v8
-; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v8
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v1, v9
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v2, v8
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v3, v1
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v4, v2
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v5, v1
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v6, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v10, v10, v10
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v9
+; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v4, v4, v12, v5
+; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v5, v6, v11, v7
+; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v8, v1
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v1, v2, v10, v3
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v4, v5
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v7, v1
+; GFX12-GISEL-FAKE16-NEXT: v_max3_num_f16 v0, v0, v1, v2
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call half @llvm.vector.reduce.fmax.v16half(<16 x half> %v)
@@ -3869,6 +4182,9 @@ declare double @llvm.vector.reduce.fmax.v4double(<4 x double>)
declare double @llvm.vector.reduce.fmax.v8double(<8 x double>)
declare double @llvm.vector.reduce.fmax.v16double(<16 x double>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10: {{.*}}
; GFX11: {{.*}}
; GFX1170: {{.*}}
; GFX12: {{.*}}
+; GFX8: {{.*}}
+; GFX9: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmin.ll
index 746b93daaf5b5..ba0d1686155c4 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmin.ll
@@ -1,59 +1,93 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1170 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1170 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1170 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1170 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1170 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define half @test_vector_reduce_fmin_v2half(<2 x half> %v) {
-; GFX7-LABEL: test_vector_reduce_fmin_v2half:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_min_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_fmin_v2half:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-SDAG-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_fmin_v2half:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-GISEL-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_fmin_v2half:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_fmin_v2half:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_fmin_v2half:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX9-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_fmin_v2half:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX10-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_fmin_v2half:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e32 v1, v0, v0
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_fmin_v2half:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX9-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_fmin_v2half:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_max_f16_e32 v1, v0, v0
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_min_f16_e32 v0, v1, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_fmin_v2half:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX10-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_fmin_v2half:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_max_f16_e32 v1, v0, v0
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_min_f16_e32 v0, v1, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmin_v2half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -204,38 +238,66 @@ define half @test_vector_reduce_fmin_v3half(<3 x half> %v) {
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_fmin_v3half:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v2
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_fmin_v3half:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX9-NEXT: s_movk_i32 s0, 0x7e00
-; GFX9-NEXT: v_pack_b32_f16 v1, v1, s0
-; GFX9-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_pk_min_f16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_fmin_v3half:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX10-NEXT: v_pack_b32_f16 v1, v1, 0x7e00
-; GFX10-NEXT: v_pk_min_f16 v0, v0, v1
-; GFX10-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_fmin_v3half:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_fmin_v3half:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e32 v2, v0, v0
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_fmin_v3half:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX9-SDAG-NEXT: s_movk_i32 s0, 0x7e00
+; GFX9-SDAG-NEXT: v_pack_b32_f16 v1, v1, s0
+; GFX9-SDAG-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_pk_min_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_fmin_v3half:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_max_f16_e32 v2, v0, v0
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX9-GISEL-NEXT: v_min3_f16 v0, v2, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_fmin_v3half:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX10-SDAG-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX10-SDAG-NEXT: v_pack_b32_f16 v1, v1, 0x7e00
+; GFX10-SDAG-NEXT: v_pk_min_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_fmin_v3half:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_max_f16_e32 v2, v0, v0
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX10-GISEL-NEXT: v_min3_f16 v0, v2, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmin_v3half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -274,14 +336,12 @@ define half @test_vector_reduce_fmin_v3half(<3 x half> %v) {
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmin_v3half:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, 0x7e00
-; GFX11-GISEL-FAKE16-NEXT: v_pk_min_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v0, v0, v2, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-SDAG-TRUE16-LABEL: test_vector_reduce_fmin_v3half:
@@ -321,14 +381,12 @@ define half @test_vector_reduce_fmin_v3half(<3 x half> %v) {
; GFX1170-GISEL-FAKE16-LABEL: test_vector_reduce_fmin_v3half:
; GFX1170-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v0
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, 0x7e00
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_min_num_f16 v0, v0, v1
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX1170-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v0, v0, v1
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v2, v1
; GFX1170-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_fmin_v3half:
@@ -384,14 +442,12 @@ define half @test_vector_reduce_fmin_v3half(<3 x half> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v0
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_pack_b32_f16 v1, v1, 0x7e00
-; GFX12-GISEL-FAKE16-NEXT: v_pk_min_num_f16 v0, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v2, v1
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call half @llvm.vector.reduce.fmin.v3half(<3 x half> %v)
@@ -437,37 +493,71 @@ define half @test_vector_reduce_fmin_v4half(<4 x half> %v) {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_fmin_v4half:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v3, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX8-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v3
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_fmin_v4half:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_max_f16 v1, v1, v1
-; GFX9-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_pk_min_f16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_fmin_v4half:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v1, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX10-NEXT: v_pk_min_f16 v0, v0, v1
-; GFX10-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_fmin_v4half:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v3, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v3
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_fmin_v4half:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e32 v2, v0, v0
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_max_f16_e32 v2, v1, v1
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v1, v2, v1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_fmin_v4half:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_pk_max_f16 v1, v1, v1
+; GFX9-SDAG-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_pk_min_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_fmin_v4half:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_max_f16_e32 v3, v1, v1
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v2, v0, v0
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_min_f16_e32 v1, v3, v1
+; GFX9-GISEL-NEXT: v_min3_f16 v0, v2, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_fmin_v4half:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_pk_max_f16 v1, v1, v1
+; GFX10-SDAG-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX10-SDAG-NEXT: v_pk_min_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_fmin_v4half:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_max_f16_e32 v2, v1, v1
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v3, v0, v0
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_min_f16_e32 v1, v2, v1
+; GFX10-GISEL-NEXT: v_min3_f16 v0, v3, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmin_v4half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -506,13 +596,16 @@ define half @test_vector_reduce_fmin_v4half(<4 x half> %v) {
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmin_v4half:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v1, v1, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_pk_min_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v1, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v0, v0, v3, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-SDAG-TRUE16-LABEL: test_vector_reduce_fmin_v4half:
@@ -552,13 +645,16 @@ define half @test_vector_reduce_fmin_v4half(<4 x half> %v) {
; GFX1170-GISEL-FAKE16-LABEL: test_vector_reduce_fmin_v4half:
; GFX1170-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v1, v1, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v0
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_min_num_f16 v0, v0, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v0, v0, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v1, v1, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v3, v1
; GFX1170-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_fmin_v4half:
@@ -614,13 +710,16 @@ define half @test_vector_reduce_fmin_v4half(<4 x half> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v1, v1, v1
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v0
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_pk_min_num_f16 v0, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v1, v1, v2
+; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v3, v1
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call half @llvm.vector.reduce.fmin.v4half(<4 x half> %v)
@@ -702,53 +801,109 @@ define half @test_vector_reduce_fmin_v8half(<8 x half> %v) {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_fmin_v8half:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX8-NEXT: v_max_f16_sdwa v4, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v5
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v4
-; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v3, v3
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_fmin_v8half:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_max_f16 v3, v3, v3
-; GFX9-NEXT: v_pk_max_f16 v1, v1, v1
-; GFX9-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX9-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX9-NEXT: v_pk_min_f16 v1, v1, v3
-; GFX9-NEXT: v_pk_min_f16 v0, v0, v2
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_pk_min_f16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_fmin_v8half:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v3, v3, v3
-; GFX10-NEXT: v_pk_max_f16 v1, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v0, v0, v0
-; GFX10-NEXT: v_pk_min_f16 v1, v1, v3
-; GFX10-NEXT: v_pk_min_f16 v0, v0, v2
-; GFX10-NEXT: v_pk_min_f16 v0, v0, v1
-; GFX10-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_fmin_v8half:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v5, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v4, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v5
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v2, v2
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v3, v3
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_fmin_v8half:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e32 v4, v0, v0
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v4, v0
+; GFX8-GISEL-NEXT: v_max_f16_e32 v4, v1, v1
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v1, v4, v1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v4, v2, v2
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v2, v4, v2
+; GFX8-GISEL-NEXT: v_max_f16_e32 v4, v3, v3
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v3, v4, v3
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v1, v2, v3
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_fmin_v8half:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_pk_max_f16 v3, v3, v3
+; GFX9-SDAG-NEXT: v_pk_max_f16 v1, v1, v1
+; GFX9-SDAG-NEXT: v_pk_max_f16 v2, v2, v2
+; GFX9-SDAG-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX9-SDAG-NEXT: v_pk_min_f16 v1, v1, v3
+; GFX9-SDAG-NEXT: v_pk_min_f16 v0, v0, v2
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_pk_min_f16 v0, v0, v1
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_fmin_v8half:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_max_f16_e32 v5, v1, v1
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v4, v0, v0
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_min_f16_e32 v1, v5, v1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v5, v2, v2
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_min_f16_e32 v3, v6, v3
+; GFX9-GISEL-NEXT: v_min3_f16 v0, v4, v0, v1
+; GFX9-GISEL-NEXT: v_min3_f16 v1, v5, v2, v3
+; GFX9-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_fmin_v8half:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_pk_max_f16 v3, v3, v3
+; GFX10-SDAG-NEXT: v_pk_max_f16 v1, v1, v1
+; GFX10-SDAG-NEXT: v_pk_max_f16 v2, v2, v2
+; GFX10-SDAG-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX10-SDAG-NEXT: v_pk_min_f16 v1, v1, v3
+; GFX10-SDAG-NEXT: v_pk_min_f16 v0, v0, v2
+; GFX10-SDAG-NEXT: v_pk_min_f16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_fmin_v8half:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_max_f16_e32 v4, v1, v1
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v5, v0, v0
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_min_f16_e32 v1, v4, v1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v4, v2, v2
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_min_f16_e32 v3, v6, v3
+; GFX10-GISEL-NEXT: v_min3_f16 v0, v5, v0, v1
+; GFX10-GISEL-NEXT: v_min3_f16 v1, v4, v2, v3
+; GFX10-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmin_v8half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -805,16 +960,23 @@ define half @test_vector_reduce_fmin_v8half(<8 x half> %v) {
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmin_v8half:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v3, v3, v3
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v1, v1, v1
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v2, v2, v2
-; GFX11-GISEL-FAKE16-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v7, v7, v7
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v1, v1, v5
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v3, v3, v7
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_pk_min_f16 v1, v1, v3
-; GFX11-GISEL-FAKE16-NEXT: v_pk_min_f16 v0, v0, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_pk_min_f16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v0, v0, v4, v1
+; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v1, v2, v5, v3
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -874,16 +1036,23 @@ define half @test_vector_reduce_fmin_v8half(<8 x half> %v) {
; GFX1170-GISEL-FAKE16-LABEL: test_vector_reduce_fmin_v8half:
; GFX1170-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v3, v3, v3
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v1, v1, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v2, v2, v2
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v7, v7, v7
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
+; GFX1170-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v1, v1, v5
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v6, v6
+; GFX1170-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v7
; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_min_num_f16 v1, v1, v3
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_min_num_f16 v0, v0, v2
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_min_num_f16 v0, v0, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v4, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v1, v2, v5, v3
; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1170-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v0, v0, v1
; GFX1170-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -959,16 +1128,23 @@ define half @test_vector_reduce_fmin_v8half(<8 x half> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v3, v3, v3
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v1, v1, v1
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v2, v2, v2
-; GFX12-GISEL-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v7, v7, v7
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
+; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v1, v1, v5
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v6, v6
+; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v7
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-FAKE16-NEXT: v_pk_min_num_f16 v1, v1, v3
-; GFX12-GISEL-FAKE16-NEXT: v_pk_min_num_f16 v0, v0, v2
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_pk_min_num_f16 v0, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v4, v1
+; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v1, v2, v5, v3
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v0, v0, v1
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1124,85 +1300,183 @@ define half @test_vector_reduce_fmin_v16half(<16 x half> %v) {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_fmin_v16half:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v9, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX8-NEXT: v_max_f16_sdwa v8, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v9
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v8
-; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v3, v3
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v4, v4
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v5, v5
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v6, v6
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v7, v7
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f16_sdwa v1, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: test_vector_reduce_fmin_v16half:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_max_f16_sdwa v15, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v14, 16, v1
-; GFX9-NEXT: v_min_f16_e32 v0, v0, v15
-; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v2
-; GFX9-NEXT: v_min3_f16 v0, v0, v1, v14
-; GFX9-NEXT: v_lshrrev_b32_e32 v12, 16, v3
-; GFX9-NEXT: v_min3_f16 v0, v0, v2, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v4
-; GFX9-NEXT: v_min3_f16 v0, v0, v3, v12
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v5
-; GFX9-NEXT: v_min3_f16 v0, v0, v4, v11
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v6
-; GFX9-NEXT: v_min3_f16 v0, v0, v5, v10
-; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; GFX9-NEXT: v_min3_f16 v0, v0, v6, v9
-; GFX9-NEXT: v_min3_f16 v0, v0, v7, v8
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: test_vector_reduce_fmin_v16half:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_max_f16_sdwa v8, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX10-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; GFX10-NEXT: v_min_f16_e32 v0, v0, v8
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX10-NEXT: v_min3_f16 v0, v0, v1, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX10-NEXT: v_min3_f16 v0, v0, v2, v8
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX10-NEXT: v_min3_f16 v0, v0, v3, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; GFX10-NEXT: v_min3_f16 v0, v0, v4, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; GFX10-NEXT: v_min3_f16 v0, v0, v5, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX10-NEXT: v_min3_f16 v0, v0, v6, v2
-; GFX10-NEXT: v_min3_f16 v0, v0, v7, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_fmin_v16half:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v9, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v8, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v9
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v2, v2
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v3, v3
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v4, v4
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v5, v5
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v6, v6
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v7, v7
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v1, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_fmin_v16half:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v0, v0
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v8, v0
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v1, v1
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v1, v8, v1
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v2, v2
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v2, v8, v2
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v3, v3
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v3, v8, v3
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v4, v4
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v4, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v4, v8, v4
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v5, v5
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v5, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v5, v8, v5
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v6, v6
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v6, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v6, v8, v6
+; GFX8-GISEL-NEXT: v_max_f16_e32 v8, v7, v7
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v7, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v7, v8, v7
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v1, v2, v3
+; GFX8-GISEL-NEXT: v_min_f16_e32 v2, v4, v5
+; GFX8-GISEL-NEXT: v_min_f16_e32 v3, v6, v7
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_min_f16_e32 v1, v2, v3
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_fmin_v16half:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_max_f16_sdwa v15, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v14, 16, v1
+; GFX9-SDAG-NEXT: v_min_f16_e32 v0, v0, v15
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v13, 16, v2
+; GFX9-SDAG-NEXT: v_min3_f16 v0, v0, v1, v14
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v12, 16, v3
+; GFX9-SDAG-NEXT: v_min3_f16 v0, v0, v2, v13
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v11, 16, v4
+; GFX9-SDAG-NEXT: v_min3_f16 v0, v0, v3, v12
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX9-SDAG-NEXT: v_min3_f16 v0, v0, v4, v11
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v9, 16, v6
+; GFX9-SDAG-NEXT: v_min3_f16 v0, v0, v5, v10
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-SDAG-NEXT: v_min3_f16 v0, v0, v6, v9
+; GFX9-SDAG-NEXT: v_min3_f16 v0, v0, v7, v8
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_fmin_v16half:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_max_f16_e32 v9, v1, v1
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v10, v3, v3
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v11, v5, v5
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v5, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v12, v7, v7
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v7, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v8, v0, v0
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_min_f16_e32 v1, v9, v1
+; GFX9-GISEL-NEXT: v_max_f16_e32 v9, v2, v2
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_min_f16_e32 v3, v10, v3
+; GFX9-GISEL-NEXT: v_max_f16_e32 v10, v4, v4
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v4, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_min_f16_e32 v5, v11, v5
+; GFX9-GISEL-NEXT: v_max_f16_e32 v11, v6, v6
+; GFX9-GISEL-NEXT: v_max_f16_sdwa v6, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_min_f16_e32 v7, v12, v7
+; GFX9-GISEL-NEXT: v_min3_f16 v0, v8, v0, v1
+; GFX9-GISEL-NEXT: v_min3_f16 v1, v9, v2, v3
+; GFX9-GISEL-NEXT: v_min3_f16 v2, v10, v4, v5
+; GFX9-GISEL-NEXT: v_min3_f16 v3, v11, v6, v7
+; GFX9-GISEL-NEXT: v_min_f16_e32 v2, v2, v3
+; GFX9-GISEL-NEXT: v_min3_f16 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_fmin_v16half:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_max_f16_sdwa v8, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-SDAG-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX10-SDAG-NEXT: v_min_f16_e32 v0, v0, v8
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX10-SDAG-NEXT: v_min3_f16 v0, v0, v1, v9
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; GFX10-SDAG-NEXT: v_min3_f16 v0, v0, v2, v8
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; GFX10-SDAG-NEXT: v_min3_f16 v0, v0, v3, v1
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; GFX10-SDAG-NEXT: v_min3_f16 v0, v0, v4, v2
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX10-SDAG-NEXT: v_min3_f16 v0, v0, v5, v1
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX10-SDAG-NEXT: v_min3_f16 v0, v0, v6, v2
+; GFX10-SDAG-NEXT: v_min3_f16 v0, v0, v7, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_fmin_v16half:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_max_f16_e32 v8, v1, v1
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v10, v5, v5
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v5, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v12, v7, v7
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v7, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_min_f16_e32 v1, v8, v1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v8, v3, v3
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v11, v4, v4
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v4, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_min_f16_e32 v5, v10, v5
+; GFX10-GISEL-NEXT: v_max_f16_e32 v10, v6, v6
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v6, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_min_f16_e32 v7, v12, v7
+; GFX10-GISEL-NEXT: v_max_f16_e32 v9, v0, v0
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_max_f16_e32 v12, v2, v2
+; GFX10-GISEL-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-GISEL-NEXT: v_min_f16_e32 v3, v8, v3
+; GFX10-GISEL-NEXT: v_min3_f16 v4, v11, v4, v5
+; GFX10-GISEL-NEXT: v_min3_f16 v5, v10, v6, v7
+; GFX10-GISEL-NEXT: v_min3_f16 v0, v9, v0, v1
+; GFX10-GISEL-NEXT: v_min3_f16 v1, v12, v2, v3
+; GFX10-GISEL-NEXT: v_min_f16_e32 v2, v4, v5
+; GFX10-GISEL-NEXT: v_min3_f16 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_fmin_v16half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1287,29 +1561,42 @@ define half @test_vector_reduce_fmin_v16half(<16 x half> %v) {
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmin_v16half:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v9, v9, v9
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v7, v7, v7
+; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v1, v1, v9
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v9, v11, v11
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v11, v13, v13
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v13, v15, v15
; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v12, v12, v12
+; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v5, v5, v11
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v6, v6, v6
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v11, v14, v14
+; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v7, v7, v13
; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v8, v8, v8
-; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v0, v0, v8
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v0, v0, v1, v9
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v0, v0, v2, v8
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v0, v0, v3, v1
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v0, v0, v4, v2
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v0, v0, v5, v1
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v0, v0, v6, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-GISEL-FAKE16-NEXT: v_max_f16_e32 v10, v10, v10
+; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v3, v3, v9
+; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v4, v4, v12, v5
+; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v5, v6, v11, v7
+; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v0, v0, v8, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v1, v2, v10, v3
+; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v2, v4, v5
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v0, v0, v7, v1
+; GFX11-GISEL-FAKE16-NEXT: v_min3_f16 v0, v0, v1, v2
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-SDAG-TRUE16-LABEL: test_vector_reduce_fmin_v16half:
@@ -1395,29 +1682,42 @@ define half @test_vector_reduce_fmin_v16half(<16 x half> %v) {
; GFX1170-GISEL-FAKE16-LABEL: test_vector_reduce_fmin_v16half:
; GFX1170-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v9, v9, v9
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v7, v7, v7
+; GFX1170-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v1, v1, v9
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v9, v11, v11
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v11, v13, v13
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v13, v15, v15
; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v12, v12, v12
+; GFX1170-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v11
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v6, v6, v6
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v11, v14, v14
+; GFX1170-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v7, v7, v13
; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v8, v8, v8
-; GFX1170-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v0, v0, v8
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v1, v9
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v2, v8
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v3, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v4, v2
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v5, v1
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v6, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX1170-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v10, v10, v10
+; GFX1170-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v9
+; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v4, v4, v12, v5
+; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v5, v6, v11, v7
+; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v8, v1
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v1, v2, v10, v3
+; GFX1170-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v2, v4, v5
; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v7, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v1, v2
; GFX1170-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_fmin_v16half:
@@ -1519,29 +1819,42 @@ define half @test_vector_reduce_fmin_v16half(<16 x half> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v9, v9, v9
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v7, v7, v7
+; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v1, v1, v9
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v9, v11, v11
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v11, v13, v13
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v13, v15, v15
; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v12, v12, v12
+; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v11
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v6, v6, v6
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v11, v14, v14
+; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v7, v7, v13
; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v8, v8, v8
-; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v0, v0, v8
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v1, v9
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v2, v8
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v3, v1
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v4, v2
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v5, v1
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v6, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-GISEL-FAKE16-NEXT: v_max_num_f16_e32 v10, v10, v10
+; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v9
+; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v4, v4, v12, v5
+; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v5, v6, v11, v7
+; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v8, v1
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v1, v2, v10, v3
+; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v2, v4, v5
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v7, v1
+; GFX12-GISEL-FAKE16-NEXT: v_min3_num_f16 v0, v0, v1, v2
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call half @llvm.vector.reduce.fmin.v16half(<16 x half> %v)
@@ -3868,6 +4181,9 @@ declare double @llvm.vector.reduce.fmin.v4double(<4 x double>)
declare double @llvm.vector.reduce.fmin.v8double(<8 x double>)
declare double @llvm.vector.reduce.fmin.v16double(<16 x double>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10: {{.*}}
; GFX11: {{.*}}
; GFX1170: {{.*}}
; GFX12: {{.*}}
+; GFX8: {{.*}}
+; GFX9: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fminimum.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fminimum.ll
index c18a41402fd56..4d0a25e516022 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fminimum.ll
@@ -8,11 +8,11 @@
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1170 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1170 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-SDAG,GFX1170-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1170 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1170 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1170 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1170,GFX1170-GISEL,GFX1170-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define half @test_vector_reduce_fminimum_v2half(<2 x half> %v) {
; GFX7-LABEL: test_vector_reduce_fminimum_v2half:
@@ -270,13 +270,9 @@ define half @test_vector_reduce_fminimum_v3half(<3 x half> %v) {
; GFX1170-GISEL-FAKE16-LABEL: test_vector_reduce_fminimum_v3half:
; GFX1170-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX1170-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x7c00
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX1170-GISEL-FAKE16-NEXT: v_pk_minimum_f16 v0, v0, v1
-; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX1170-GISEL-FAKE16-NEXT: v_minimum_f16 v0, v0, v1
+; GFX1170-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX1170-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-GISEL-FAKE16-NEXT: v_minimum3_f16 v0, v0, v2, v1
; GFX1170-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_fminimum_v3half:
@@ -326,14 +322,9 @@ define half @test_vector_reduce_fminimum_v3half(<3 x half> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x7c00
-; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-FAKE16-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_pk_minimum_f16 v0, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_minimum_f16 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: v_minimum3_f16 v0, v0, v2, v1
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call half @llvm.vector.reduce.fminimum.v3half(<3 x half> %v)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
index c77bee085ce47..0ae1cbde1e76e 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
@@ -232,10 +232,10 @@ define half @test_vector_reduce_fmul_v3half(half %sp, <3 x half> %v) {
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmul_v3half:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v3
; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v2
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -286,10 +286,10 @@ define half @test_vector_reduce_fmul_v3half(half %sp, <3 x half> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX12-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX12-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v3
; GFX12-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v2
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
index 7e6bbacea862a..41703e7ef6ac5 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
@@ -1,21 +1,21 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; FIXME-TRUE16. enable gisel
; XUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; XUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define i8 @test_vector_reduce_mul_v2i8(<2 x i8> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v2i8:
@@ -969,19 +969,34 @@ entry:
}
define i16 @test_vector_reduce_mul_v2i16(<2 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_mul_v2i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_mul_lo_u32 v0, v0, v1
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_mul_v2i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_mul_v2i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_mul_v2i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_mul_v2i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_mul_v2i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_vector_reduce_mul_v2i16:
; GFX9: ; %bb.0: ; %entry
@@ -1088,21 +1103,36 @@ define i16 @test_vector_reduce_mul_v3i16(<3 x i16> %v) {
; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_mul_v3i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_lo_u16 v1, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_mul_lo_u16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_mul_v3i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v0, v1
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_mul_lo_u16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_mul_v3i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX10-NEXT: v_mul_lo_u16 v0, v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_mul_v3i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_mul_v3i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_mul_v3i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1125,9 +1155,9 @@ define i16 @test_vector_reduce_mul_v3i16(<3 x i16> %v) {
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v3i16:
@@ -1163,9 +1193,9 @@ define i16 @test_vector_reduce_mul_v3i16(<3 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.mul.v3i16(<3 x i16> %v)
@@ -1195,13 +1225,22 @@ define i16 @test_vector_reduce_mul_v4i16(<4 x i16> %v) {
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_mul_v4i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_mul_v4i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_mul_v4i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_vector_reduce_mul_v4i16:
; GFX9: ; %bb.0: ; %entry
@@ -1325,17 +1364,30 @@ define i16 @test_vector_reduce_mul_v8i16(<8 x i16> %v) {
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_mul_v8i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_lo_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v2, v5, v4
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_mul_v8i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v5, v4
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_mul_v8i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v1, v3
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v4, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v8i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1528,25 +1580,46 @@ define i16 @test_vector_reduce_mul_v16i16(<16 x i16> %v) {
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_mul_v16i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_lo_u16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_lo_u16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_lo_u16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_lo_u16_e32 v2, v2, v6
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v4
-; GFX8-NEXT: v_mul_lo_u16_e32 v3, v3, v7
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v1, v5
-; GFX8-NEXT: v_mul_lo_u16_e32 v4, v11, v10
-; GFX8-NEXT: v_mul_lo_u16_e32 v5, v9, v8
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v2, v5, v4
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_mul_v16i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v4, v11, v10
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v5, v9, v8
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v5, v4
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_mul_v16i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v8, v0, v4
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v4, v1, v5
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v5, v2, v6
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v6, v3, v7
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v5, v8, v5
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v4, v6
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v5, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v16i16:
; GFX9-SDAG: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
index 5f13fece9dd73..f45d1a4117a78 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
@@ -1260,26 +1260,40 @@ define i16 @test_vector_reduce_smax_v3i16(<3 x i16> %v) {
; GFX8-NEXT: v_max_i16_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_smax_v3i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s0, 0x8000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x5040100
-; GFX9-NEXT: v_perm_b32 v1, s0, v1, v2
-; GFX9-NEXT: v_pk_max_i16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_max_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_smax_v3i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s0, 0x8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v1, s0, v1, v2
+; GFX9-SDAG-NEXT: v_pk_max_i16 v0, v0, v1
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_max_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_smax_v3i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_movk_i32 s4, 0x8000
-; GFX10-NEXT: v_perm_b32 v1, s4, v1, 0x5040100
-; GFX10-NEXT: v_pk_max_i16 v0, v0, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_max_i16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_smax_v3i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_smax_v3i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX10-SDAG-NEXT: v_perm_b32 v1, s4, v1, 0x5040100
+; GFX10-SDAG-NEXT: v_pk_max_i16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-SDAG-NEXT: v_max_i16 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_smax_v3i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1305,13 +1319,9 @@ define i16 @test_vector_reduce_smax_v3i16(<3 x i16> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_smax_v3i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_movk_i32 s0, 0x8000
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX11-GISEL-NEXT: v_pk_max_i16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-GISEL-NEXT: v_max_i16 v0, v0, v1
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v3i16:
@@ -1351,14 +1361,9 @@ define i16 @test_vector_reduce_smax_v3i16(<3 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: s_movk_i32 s0, 0x8000
-; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_max_i16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_max_i16 v0, v0, v1
+; GFX12-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.smax.v3i16(<3 x i16> %v)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
index 94ac809189bd5..82021968904d8 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
@@ -1260,26 +1260,40 @@ define i16 @test_vector_reduce_smin_v3i16(<3 x i16> %v) {
; GFX8-NEXT: v_min_i16_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_smin_v3i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s0, 0x7fff
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x5040100
-; GFX9-NEXT: v_perm_b32 v1, s0, v1, v2
-; GFX9-NEXT: v_pk_min_i16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_min_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_smin_v3i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s0, 0x7fff
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v1, s0, v1, v2
+; GFX9-SDAG-NEXT: v_pk_min_i16 v0, v0, v1
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_min_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_smin_v3i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_movk_i32 s4, 0x7fff
-; GFX10-NEXT: v_perm_b32 v1, s4, v1, 0x5040100
-; GFX10-NEXT: v_pk_min_i16 v0, v0, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_min_i16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_smin_v3i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_smin_v3i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX10-SDAG-NEXT: v_perm_b32 v1, s4, v1, 0x5040100
+; GFX10-SDAG-NEXT: v_pk_min_i16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-SDAG-NEXT: v_min_i16 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_smin_v3i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1305,13 +1319,9 @@ define i16 @test_vector_reduce_smin_v3i16(<3 x i16> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_smin_v3i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_movk_i32 s0, 0x7fff
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX11-GISEL-NEXT: v_pk_min_i16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-GISEL-NEXT: v_min_i16 v0, v0, v1
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v3i16:
@@ -1351,14 +1361,9 @@ define i16 @test_vector_reduce_smin_v3i16(<3 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: s_movk_i32 s0, 0x7fff
-; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_min_i16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_min_i16 v0, v0, v1
+; GFX12-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.smin.v3i16(<3 x i16> %v)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
index 2eb06b2f2c1f5..c763593c67392 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
@@ -1184,23 +1184,37 @@ define i16 @test_vector_reduce_umax_v3i16(<3 x i16> %v) {
; GFX8-NEXT: v_max_u16_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_umax_v3i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_pk_max_u16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_max_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_umax_v3i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-SDAG-NEXT: v_pk_max_u16 v0, v0, v1
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_max_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_umax_v3i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_pk_max_u16 v0, v0, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_max_u16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_umax_v3i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_umax_v3i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-SDAG-NEXT: v_pk_max_u16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-SDAG-NEXT: v_max_u16 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_umax_v3i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1225,12 +1239,9 @@ define i16 @test_vector_reduce_umax_v3i16(<3 x i16> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_umax_v3i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_pk_max_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_max_u16 v0, v0, v1
+; GFX11-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v3i16:
@@ -1268,12 +1279,9 @@ define i16 @test_vector_reduce_umax_v3i16(<3 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_max_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_max_u16 v0, v0, v1
+; GFX12-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.umax.v3i16(<3 x i16> %v)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
index 7c3e52e778986..4849c0a9011e8 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
@@ -1314,24 +1314,38 @@ define i16 @test_vector_reduce_umin_v3i16(<3 x i16> %v) {
; GFX8-NEXT: v_min_u16_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_umin_v3i16:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x5040100
-; GFX9-NEXT: v_perm_b32 v1, -1, v1, v2
-; GFX9-NEXT: v_pk_min_u16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: test_vector_reduce_umin_v3i16:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v1, -1, v1, v2
+; GFX9-SDAG-NEXT: v_pk_min_u16 v0, v0, v1
+; GFX9-SDAG-NEXT: s_nop 0
+; GFX9-SDAG-NEXT: v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_umin_v3i16:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_perm_b32 v1, -1, v1, 0x5040100
-; GFX10-NEXT: v_pk_min_u16 v0, v0, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_min_u16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-LABEL: test_vector_reduce_umin_v3i16:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-GISEL-NEXT: v_min3_u16 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_umin_v3i16:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_perm_b32 v1, -1, v1, 0x5040100
+; GFX10-SDAG-NEXT: v_pk_min_u16 v0, v0, v1
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-SDAG-NEXT: v_min_u16 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_umin_v3i16:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-GISEL-NEXT: v_min3_u16 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umin_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -1356,12 +1370,9 @@ define i16 @test_vector_reduce_umin_v3i16(<3 x i16> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_umin_v3i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_perm_b32 v1, -1, v1, 0x5040100
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_pk_min_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_min_u16 v0, v0, v1
+; GFX11-GISEL-NEXT: v_min3_u16 v0, v0, v2, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_umin_v3i16:
@@ -1399,12 +1410,9 @@ define i16 @test_vector_reduce_umin_v3i16(<3 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_perm_b32 v1, -1, v1, 0x5040100
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_min_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_min_u16 v0, v0, v1
+; GFX12-GISEL-NEXT: v_min3_u16 v0, v0, v2, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.umin.v3i16(<3 x i16> %v)
More information about the llvm-branch-commits
mailing list